diff --git a/py_hamt/encryption_hamt_store.py b/py_hamt/encryption_hamt_store.py index 7e997af..c750fd5 100644 --- a/py_hamt/encryption_hamt_store.py +++ b/py_hamt/encryption_hamt_store.py @@ -183,13 +183,13 @@ async def get( return None async def set(self, key: str, value: zarr.core.buffer.Buffer) -> None: - """@private""" + """Encrypt and store a value, then update cached metadata.""" if self.read_only: raise Exception("Cannot write to a read-only store.") raw_bytes = value.to_bytes() - if key in self.metadata_read_cache: - self.metadata_read_cache[key] = raw_bytes # Encrypt it encrypted_bytes = self._encrypt(raw_bytes) await self.hamt.set(key, encrypted_bytes) + if key in self.metadata_read_cache: + self.metadata_read_cache[key] = raw_bytes diff --git a/py_hamt/hamt.py b/py_hamt/hamt.py index 0edbba9..0a54665 100644 --- a/py_hamt/hamt.py +++ b/py_hamt/hamt.py @@ -14,12 +14,14 @@ ) import dag_cbor +from blake3 import blake3 from dag_cbor.ipld import IPLDKind -from multiformats import multihash from . import instrumentation from .store_httpx import ContentAddressedStore +_VACATE_CONCURRENCY = 16 + def extract_bits(hash_bytes: bytes, depth: int, nbits: int) -> int: """ @@ -48,18 +50,9 @@ def extract_bits(hash_bytes: bytes, depth: int, nbits: int) -> int: return result -b3 = multihash.get("blake3") - - def blake3_hashfn(input_bytes: bytes) -> bytes: - """ - This is the default blake3 hash function used for the `HAMT`, with a 32 byte hash size. - - """ - # 32 bytes is the recommended byte size for blake3 and the default, but multihash forces us to explicitly specify - digest: bytes = b3.digest(input_bytes, size=32) - raw_bytes: bytes = b3.unwrap(digest) - return raw_bytes + """Return the HAMT's default raw 32-byte BLAKE3 digest.""" + return blake3(input_bytes).digest(length=32) class Node: @@ -197,6 +190,32 @@ def __init__(self, hamt: "HAMT") -> None: self.hamt: HAMT = hamt # The integer key is a uuidv4 128-bit integer, for (almost perfectly) guaranteeing uniqueness self.buffer: dict[int, Node] = {} + # CAS IDs identify immutable data, so clean nodes can be reused until the + # cache is explicitly vacated. Dirty nodes remain exclusively in buffer. + self.clean_cache: dict[IPLDKind, Node] = {} + + def get_clean_node(self, node_id: IPLDKind) -> Node | None: + """Return a cached CAS node, or None for misses and unhashable IDs.""" + try: + return self.clean_cache.get(node_id) + except TypeError: + return None + + def cache_clean_node(self, node_id: IPLDKind, node: Node) -> None: + """Cache a CAS node when its IPLD identifier is hashable.""" + try: + self.clean_cache[node_id] = node + except TypeError: + # CAS implementations normally use bytes or CID objects. Gracefully + # skip caching for a custom store that returns another IPLD kind. + pass + + def remove_clean_node(self, node_id: IPLDKind) -> None: + """Stop serving a clean node once that object becomes dirty.""" + try: + self.clean_cache.pop(node_id, None) + except TypeError: + pass def is_buffer_id(self, id: IPLDKind) -> bool: return id in self.buffer @@ -223,54 +242,85 @@ def size(self) -> int: copied.set_link(link_index, InMemoryTreeStore._replaced_id_marker) total += len(copied.serialize()) + # Clean cached nodes contain only serializable CAS links and therefore do + # not need the buffer-ID substitution used for dirty nodes above. + for node in self.clean_cache.values(): + total += len(node.serialize()) + return total - # The HAMT must properly acquire a lock for this to run successfully! This is not async or thread safe - # This algorithm has an implicit assumption that the entire continuous line of branches up to an ancestor is in the memory buffer, which will happen because of the DFS style traversal in all HAMT operations async def vacate(self) -> None: - # node stack is a list of tuples that look like (parent_id, self_id, node) - node_stack: list[tuple[int | None, int, Node]] = [] + """Flush the dirty tree in concurrent, bottom-up sibling waves. + + The HAMT lock must be held by the caller. A node is not serialized until + every buffered child has been saved and relinked to its CAS ID, so only + independent subtrees overlap and the root is always saved last. + """ # The root node may not be in the buffer, e.g. this is HAMT initialized with a specific root node id if self.is_buffer_id(self.hamt.root_node_id): - root_node: Node = self.buffer[cast(int, self.hamt.root_node_id)] - node_stack.append((None, cast(int, self.hamt.root_node_id), root_node)) - - while len(node_stack) > 0: - parent_buffer_id, top_buffer_id, top_node = node_stack[-1] - new_nodes_on_stack: list[tuple[int, int, Node]] = [] - for child_buffer_id in self.children_in_memory(top_node): - child_node: Node = self.buffer[child_buffer_id] - new_nodes_on_stack.append((top_buffer_id, child_buffer_id, child_node)) - - no_children_in_memory: bool = len(new_nodes_on_stack) == 0 - # Flush this node out and relink the rest of the tree - if no_children_in_memory: - is_root: bool = parent_buffer_id is None - old_id: int = top_buffer_id - new_id: IPLDKind = await self.hamt.cas.save( - top_node.serialize(), codec="dag-cbor" - ) - del self.buffer[old_id] - node_stack.pop() - - # If it's the root, we need to set the hamt's root node id once this is done sending to the backing store - if is_root: - self.hamt.root_node_id = new_id - # Edit and properly relink the parent if this is not the root - else: - # parent_buffer_id is never None in this branch - assert parent_buffer_id is not None - parent_node: Node = self.buffer[parent_buffer_id] - parent_node.replace_link(old_id, new_id) - # Continue recursing down the tree - else: - node_stack.extend(new_nodes_on_stack) + root_buffer_id = cast(int, self.hamt.root_node_id) + parent_ids: dict[int, int | None] = {root_buffer_id: None} + buffered_children: dict[int, set[int]] = {} + traversal_order: list[int] = [] + node_stack: list[int] = [root_buffer_id] + + while node_stack: + buffer_id = node_stack.pop() + traversal_order.append(buffer_id) + child_ids = set(self.children_in_memory(self.buffer[buffer_id])) + buffered_children[buffer_id] = child_ids + for child_id in child_ids: + parent_ids[child_id] = buffer_id + node_stack.append(child_id) + + remaining_ids = set(traversal_order) + while remaining_ids: + wave_ids = [ + buffer_id + for buffer_id in traversal_order + if buffer_id in remaining_ids + and buffered_children[buffer_id].isdisjoint(remaining_ids) + ] + new_ids: list[IPLDKind] = [None] * len(wave_ids) + next_wave_index = 0 + + async def save_nodes() -> None: + nonlocal next_wave_index + while next_wave_index < len(wave_ids): + wave_index = next_wave_index + next_wave_index += 1 + buffer_id = wave_ids[wave_index] + new_ids[wave_index] = await self.hamt.cas.save( + self.buffer[buffer_id].serialize(), codec="dag-cbor" + ) + + save_tasks = [ + asyncio.ensure_future(save_nodes()) + for _ in range(min(_VACATE_CONCURRENCY, len(wave_ids))) + ] + try: + await asyncio.gather(*save_tasks) + except BaseException: + for save_task in save_tasks: + save_task.cancel() + await asyncio.gather(*save_tasks, return_exceptions=True) + raise + + for old_id, new_id in zip(wave_ids, new_ids): + parent_id = parent_ids[old_id] + del self.buffer[old_id] + remaining_ids.remove(old_id) + if parent_id is None: + self.hamt.root_node_id = new_id + else: + self.buffer[parent_id].replace_link(old_id, new_id) # There are only two types of nodes left in the buffer: # 1. A bunch of nodes that seem "unlinked" to anything else, since Links used within the Nodes will reference the real underlying CAS # 2. A bunch of empty nodes leftover if the key values they contain are deleted, these would normally be consolidated by a content addressed system but will be leftover in the buffer # So we can just clear out everything since these nodes are not used by the rest of the tree self.buffer = {} + self.clean_cache = {} async def add_to_buffer(self, node: Node) -> IPLDKind: # This buffer_id is IPLDKind type since technically it's an int, but it's not dag_cbor serializable since that library can only do up to 64-bit ints. Thus this will throw errors early if a node is written with buffer_ids still in there @@ -283,6 +333,10 @@ async def save(self, original_id: IPLDKind, node: Node) -> IPLDKind: if self.is_buffer_id(original_id): return original_id + # The caller may have modified a node returned by load(). Do not leave + # that object cached under the CID of its original immutable contents. + self.remove_clean_node(original_id) + # This node was not in the buffer, don't save it to the backing store but rather add to it to the in memory buffer buffer_id: IPLDKind = await self.add_to_buffer(node) return buffer_id @@ -293,9 +347,13 @@ async def load(self, id: IPLDKind) -> Node: node: Node = self.buffer[cast(int, id)] # we know all buffer ids are ints return node - # Something that isn't in the in memory tree, add it + clean_node = self.get_clean_node(id) + if clean_node is not None: + return clean_node + + # CAS-loaded nodes are clean and keyed by their immutable content ID. node = Node.deserialize(await self.hamt.cas.load(id)) - await self.add_to_buffer(node) + self.cache_clean_node(id, node) return node @@ -424,9 +482,12 @@ async def make_read_only(self) -> None: async def enable_write(self) -> None: """ - Enable both reads and writes. This creates an internal structure for performance optimizations which will result in the root node ID no longer being valid, in order to read that at the end of your operations you must first use `make_read_only`. + Enable both reads and writes. Calling this while writes are already enabled is a no-op that preserves any buffered changes. The read-only to writable transition creates an internal structure for performance optimizations which will result in the root node ID no longer being valid; to read it at the end of your operations, first use `make_read_only`. """ async with self.lock: + if not self.read_only: + return + # The read cache has no writes that need to be sent upstream so we can remove it without vacating self.read_only = False self.node_store = InMemoryTreeStore(self) @@ -461,12 +522,16 @@ async def cache_vacate(self) -> None: await self.node_store.vacate() async def _reserialize_and_link( - self, node_stack: list[tuple[IPLDKind, Node]] + self, + node_stack: list[tuple[IPLDKind, Node]], + link_path: list[int], ) -> None: """ This function starts from the node at the end of the list and reserializes so that each node holds valid new IDs after insertion into the store Takes a stack of nodes, we represent a stack with a list where the first element is the root element and the last element is the top of the stack Each element in the list is a tuple where the first element is the ID from the store and the second element is the Node in python + `link_path[i]` is the index in `node_stack[i - 1]` through which + `node_stack[i]` is linked. The root entry at index zero is a sentinel. If a node ends up being empty, then it is deleted entirely, unless it is the root node Modifies in place """ @@ -477,18 +542,13 @@ async def _reserialize_and_link( # If this node is empty, and it's not the root node, then we can delete it entirely from the list is_root: bool = stack_index == 0 if node.is_empty() and not is_root: - # Unlink from the rest of the tree + # Unlink from the rest of the tree using the recorded parent slot. _, prev_node = node_stack[stack_index - 1] - # When removing links, don't worry about two nodes having the same link since all nodes are guaranteed to be different by the removal of empty nodes after every single operation - for link_index in prev_node.iter_link_indices(): - link = prev_node.get_link(link_index) - if link == old_id: - # Delete the link by making it an empty bucket - prev_node.data[link_index] = {} - break + prev_node.data[link_path[stack_index]] = {} # Remove from our stack, continue reserializing up the tree node_stack.pop(stack_index) + link_path.pop(stack_index) continue # If not an empty node, just reserialize like normal and replace this one @@ -498,7 +558,54 @@ async def _reserialize_and_link( # If this is not the last i.e. root node, we need to change the linking of the node prior in the list since we just reserialized if not is_root: _, prev_node = node_stack[stack_index - 1] - prev_node.replace_link(old_id, new_store_id) + prev_node.set_link(link_path[stack_index], new_store_id) + + async def _collect_subtree_entries( + self, node: Node, limit: int + ) -> dict[str, IPLDKind] | None: + """Collect a subtree's entries when they fit in a single bucket.""" + entries: dict[str, IPLDKind] = {} + + for bucket in node.iter_buckets(): + if len(entries) + len(bucket) > limit: + return None + entries.update(bucket) + + for link in node.iter_links(): + child = await self.node_store.load(link) + child_entries = await self._collect_subtree_entries( + child, limit - len(entries) + ) + if child_entries is None: + return None + entries.update(child_entries) + + return entries + + async def _collapse_delete_path( + self, + node_stack: list[tuple[IPLDKind, Node]], + link_path: list[int], + ) -> None: + """Collapse small subtrees into parent buckets after a deletion. + + Applying this bottom-up restores the same shape produced by a fresh build: + every linked subtree contains more entries than ``max_bucket_size``. This + can change CIDs only for trees that previously retained a non-canonical + post-delete shape. + """ + node_store = cast(InMemoryTreeStore, self.node_store) + for stack_index in range(len(node_stack) - 1, 0, -1): + old_id, node = node_stack[stack_index] + entries = await self._collect_subtree_entries(node, self.max_bucket_size) + if entries is None: + continue + + _, parent = node_stack[stack_index - 1] + parent.data[link_path[stack_index]] = entries + node_store.remove_clean_node(old_id) + node_stack.pop(stack_index) + link_path.pop(stack_index) # automatically skip encoding if the value provided is of the bytes variety async def set(self, key: str, val: IPLDKind) -> None: @@ -517,55 +624,105 @@ async def set(self, key: str, val: IPLDKind) -> None: pointer: IPLDKind = await self.cas.save(data, codec="raw") await self._set_pointer(key, pointer) + async def _build_overflow_subtree( + self, + kvs_queue: list[tuple[str, IPLDKind, bytes]], + depth: int, + key_hashes: dict[str, bytes], + ) -> IPLDKind: + """Build a detached subtree for a full bucket. + + Every node touched here is new to the current write. Hash exhaustion can + therefore fail without exposing a partial mutation in the existing tree. + """ + root_node = Node() + root_node_id = await self.node_store.save(None, root_node) + node_stack: list[tuple[IPLDKind, Node]] = [(root_node_id, root_node)] + + while kvs_queue: + _, top_node = node_stack[-1] + curr_key, curr_val_ptr, raw_hash = kvs_queue[0] + map_key = extract_bits(raw_hash, depth + len(node_stack) - 1, 8) + + item = top_node.data[map_key] + if isinstance(item, list): + next_node_id = item[0] + next_node = await self.node_store.load(next_node_id) + node_stack.append((next_node_id, next_node)) + continue + + bucket = cast(dict[str, IPLDKind], item) + if curr_key in bucket or len(bucket) < self.max_bucket_size: + bucket[curr_key] = curr_val_ptr + kvs_queue.pop(0) + continue + + for bucket_key, bucket_value in bucket.items(): + kvs_queue.append((bucket_key, bucket_value, key_hashes[bucket_key])) + + new_node = Node() + new_node_id = await self.node_store.save(None, new_node) + top_node.set_link(map_key, new_node_id) + + return root_node_id + async def _set_pointer(self, key: str, val_ptr: IPLDKind) -> None: + """Set a value pointer without exposing partial tree mutations on failure.""" async with self.lock: + node_store = cast(InMemoryTreeStore, self.node_store) node_stack: list[tuple[IPLDKind, Node]] = [] + link_path: list[int] = [-1] root_node: Node = await self.node_store.load(self.root_node_id) node_stack.append((self.root_node_id, root_node)) - # FIFO queue to keep track of all the KVs we need to insert - # This is needed if any buckets overflow and so we need to reinsert all those KVs - kvs_queue: list[tuple[str, IPLDKind]] = [] - kvs_queue.append((key, val_ptr)) - - while len(kvs_queue) > 0: + raw_hash = self.hash_fn(key.encode()) + while True: _, top_node = node_stack[-1] - curr_key, curr_val_ptr = kvs_queue[0] - - raw_hash: bytes = self.hash_fn(curr_key.encode()) - map_key: int = extract_bits(raw_hash, len(node_stack) - 1, 8) + map_key = extract_bits(raw_hash, len(node_stack) - 1, 8) item = top_node.data[map_key] if isinstance(item, list): - next_node_id: IPLDKind = item[0] - next_node: Node = await self.node_store.load(next_node_id) + next_node_id = item[0] + next_node = await self.node_store.load(next_node_id) node_stack.append((next_node_id, next_node)) - elif isinstance(item, dict): - bucket: dict[str, IPLDKind] = item - - # If this bucket already has this same key, or has space, just rewrite the value and then go work on the others in the queue - if curr_key in bucket or len(bucket) < self.max_bucket_size: - bucket[curr_key] = curr_val_ptr - kvs_queue.pop(0) - continue + link_path.append(map_key) + continue - # The current key is not in the bucket and the bucket is too full, so empty KVs from the bucket and restart insertion - for k in bucket: - v_ptr = bucket[k] - kvs_queue.append((k, v_ptr)) - - # Create a new link to a new node so that we can reflow these KVs into a new subtree - new_node = Node() - new_node_id: IPLDKind = await self.node_store.save(None, new_node) - link: list[IPLDKind] = [new_node_id] - top_node.data[map_key] = link + bucket = cast(dict[str, IPLDKind], item) + if key in bucket or len(bucket) < self.max_bucket_size: + bucket[key] = val_ptr + break - # Finally, reserialize and fix all links, deleting empty nodes as needed - await self._reserialize_and_link(node_stack) + key_hashes = {key: raw_hash} + kvs_queue = [(key, val_ptr, raw_hash)] + for bucket_key, bucket_value in bucket.items(): + key_hashes[bucket_key] = self.hash_fn(bucket_key.encode()) + kvs_queue.append((bucket_key, bucket_value, key_hashes[bucket_key])) + + original_buffer_ids = set(node_store.buffer) + try: + new_node_id = await self._build_overflow_subtree( + kvs_queue, len(node_stack), key_hashes + ) + except BaseException: + for buffer_id in set(node_store.buffer) - original_buffer_ids: + del node_store.buffer[buffer_id] + raise + + top_node.set_link(map_key, new_node_id) + break + + # Finally, reserialize and fix all links. + await self._reserialize_and_link(node_stack, link_path) self.root_node_id = node_stack[0][0] async def delete(self, key: str) -> None: - """Delete a key-value mapping.""" + """Delete a key-value mapping. + + Failure-atomic with respect to storage errors: all fallible CAS loads + happen before any node is mutated, so if deletion raises, the + observable tree remains unchanged. + """ # Also deletes the pointer at the same time so this doesn't have a _delete_pointer duo if self.read_only: @@ -575,6 +732,7 @@ async def delete(self, key: str) -> None: raw_hash: bytes = self.hash_fn(key.encode()) node_stack: list[tuple[IPLDKind, Node]] = [] + link_path: list[int] = [-1] root_node: Node = await self.node_store.load(self.root_node_id) node_stack.append((self.root_node_id, root_node)) @@ -587,6 +745,14 @@ async def delete(self, key: str) -> None: if isinstance(item, dict): bucket = item if key in bucket: + # Collapse may inspect sibling subtrees after the bucket is + # changed. Load them first so a CAS failure cannot leave a + # shared cached node partially mutated. The pre-delete tree + # has one extra entry along this path, hence the +1 budget. + for _, path_node in node_stack[1:]: + await self._collect_subtree_entries( + path_node, self.max_bucket_size + 1 + ) del bucket[key] created_change = True # Break out since whether or not the key is in the bucket, it should have been here so either now reserialize or raise a KeyError @@ -595,10 +761,12 @@ async def delete(self, key: str) -> None: link: IPLDKind = item[0] next_node: Node = await self.node_store.load(link) node_stack.append((link, next_node)) + link_path.append(map_key) - # Finally, reserialize and fix all links, deleting empty nodes as needed + # Finally, restore the canonical shape and fix all remaining links. if created_change: - await self._reserialize_and_link(node_stack) + await self._collapse_delete_path(node_stack, link_path) + await self._reserialize_and_link(node_stack, link_path) self.root_node_id = node_stack[0][0] else: # If we didn't make a change, then this key must not exist within the HAMT @@ -710,7 +878,10 @@ async def keys(self) -> AsyncIterator[str]: """ AsyncIterator returning all keys in the HAMT. - If the HAMT is write enabled, to maintain strong consistency this will obtain an async lock and not allow any other operations to proceed. + If the HAMT is write enabled, the keys present when iteration starts are + copied while holding the async lock. The lock is released before any key + is yielded, so reads and mutations are safe between iterations and do not + affect the keys returned by an iteration already in progress. When the HAMT is in read only mode however, this can be run concurrently with get operations. """ @@ -718,9 +889,12 @@ async def keys(self) -> AsyncIterator[str]: async for k in self._keys_no_locking(): yield k else: + # Buffered nodes are mutable, so copying only the root ID would not + # isolate iteration from mutations made after a caller-visible yield. async with self.lock: - async for k in self._keys_no_locking(): - yield k + keys_snapshot = [key async for key in self._keys_no_locking()] + for key in keys_snapshot: + yield key async def _keys_no_locking(self) -> AsyncIterator[str]: async for _, node in self._iter_nodes(): @@ -732,10 +906,17 @@ async def len(self) -> int: """ Return the number of key value mappings in this HAMT. - When the HAMT is write enabled, to maintain strong consistency it will acquire a lock and thus not allow any other operations to proceed until the length is fully done being calculated. If read only, then this can be run concurrently with other operations. + When the HAMT is write enabled, keys are counted directly while holding + the async lock, without materializing a snapshot. If read only, counting + can run concurrently with other operations. """ count: int = 0 - async for _ in self.keys(): - count += 1 + if self.read_only: + async for _ in self._keys_no_locking(): + count += 1 + else: + async with self.lock: + async for _ in self._keys_no_locking(): + count += 1 return count diff --git a/py_hamt/sharded_zarr_store.py b/py_hamt/sharded_zarr_store.py index b56f38e..7d0d072 100644 --- a/py_hamt/sharded_zarr_store.py +++ b/py_hamt/sharded_zarr_store.py @@ -6,7 +6,8 @@ import time import warnings from collections import OrderedDict, defaultdict -from collections.abc import AsyncIterator, Iterable +from collections.abc import AsyncIterator, Iterable, Iterator +from contextlib import asynccontextmanager from dataclasses import dataclass from typing import ( ClassVar, @@ -34,6 +35,8 @@ SHARDED_ZARR_V1 = "sharded_zarr_v1" SHARDED_ZARR_V2 = "sharded_zarr_v2" ZARR_METADATA_SUFFIXES = ("zarr.json", ".zarray", ".zattrs", ".zgroup") +_FLUSH_CONCURRENCY = 8 +_V1_INFERENCE_CONCURRENCY = 8 ShardCacheKey = int | tuple[str, int] ShardReadMode = Literal["full", "sparse"] @@ -278,14 +281,19 @@ class MemoryBoundedLRUCache: An LRU cache that evicts items when memory usage exceeds a threshold. Memory usage is calculated using sys.getsizeof for accurate sizing. - Dirty shards (those marked for writing) are never evicted until marked clean. - All operations are thread-safe for async access using an asyncio.Lock. + Dirty shards (those marked for writing) and shards pinned by active users are + never evicted. Pins are refcounted so concurrent users can safely share a shard. + If protected shards exceed the configured budget, the cache temporarily + overflows rather than evicting data that is in use or waiting to be flushed. + Mutating operations are safe for async access using an asyncio.Lock; + ``in`` membership is a plain synchronous dict read on the event loop. """ def __init__(self, max_memory_bytes: int = 100 * 1024 * 1024): # 100MB default self.max_memory_bytes = max_memory_bytes self._cache: OrderedDict[ShardCacheKey, List[Optional[CID]]] = OrderedDict() self._dirty_shards: Set[ShardCacheKey] = set() + self._pin_counts: Dict[ShardCacheKey, int] = {} self._shard_sizes: Dict[ShardCacheKey, int] = {} self._actual_memory_usage = 0 self._cache_lock = asyncio.Lock() @@ -329,27 +337,62 @@ async def put( self._shard_sizes[shard_idx] = shard_size self._actual_memory_usage += shard_size - while ( - self._actual_memory_usage > self.max_memory_bytes - and len(self._cache) > 1 - ): - evicted = False - checked_dirty_shards = set() - while self._cache: - candidate_idx, candidate_data = self._cache.popitem(last=False) - if candidate_idx not in self._dirty_shards: - self._actual_memory_usage -= self._shard_sizes.pop( - candidate_idx, 0 - ) - evicted = True - break + self._evict_if_needed_locked() + + def _evict_if_needed_locked(self) -> None: + """Evict clean, unpinned LRU entries while the cache lock is held.""" + while ( + self._actual_memory_usage > self.max_memory_bytes and len(self._cache) > 1 + ): + candidate_idx = next( + ( + cached_idx + for cached_idx in self._cache + if cached_idx not in self._dirty_shards + and self._pin_counts.get(cached_idx, 0) == 0 + ), + None, + ) + if candidate_idx is None: + return + self._cache.pop(candidate_idx) + self._actual_memory_usage -= self._shard_sizes.pop(candidate_idx, 0) + + @asynccontextmanager + async def pin(self, shard_idx: ShardCacheKey) -> AsyncIterator[None]: + """Prevent a shard from being evicted for the duration of the context.""" + async with self._cache_lock: + self._pin_counts[shard_idx] = self._pin_counts.get(shard_idx, 0) + 1 + try: + yield + finally: + async with self._cache_lock: + remaining_pins = self._pin_counts[shard_idx] - 1 + if remaining_pins: + self._pin_counts[shard_idx] = remaining_pins + else: + del self._pin_counts[shard_idx] + self._evict_if_needed_locked() - self._cache[candidate_idx] = candidate_data - checked_dirty_shards.add(candidate_idx) - if len(checked_dirty_shards) == len(self._dirty_shards): - break - if not evicted: - break + async def update_entry( + self, shard_idx: ShardCacheKey, entry_idx: int, value: Optional[CID] + ) -> bool: + """Update one cached entry and mark its shard dirty atomically.""" + async with self._cache_lock: + shard_data = self._cache.get(shard_idx) + if shard_data is None: + raise RuntimeError(f"Shard {shard_idx} not found in cache") + if shard_data[entry_idx] == value: + return False + + old_size = self._shard_sizes[shard_idx] + shard_data[entry_idx] = value + new_size = self._get_shard_size(shard_data) + self._shard_sizes[shard_idx] = new_size + self._actual_memory_usage += new_size - old_size + self._dirty_shards.add(shard_idx) + self._evict_if_needed_locked() + return True async def mark_dirty(self, shard_idx: ShardCacheKey) -> None: """Mark a shard as dirty (should not be evicted).""" @@ -361,6 +404,7 @@ async def mark_clean(self, shard_idx: ShardCacheKey) -> None: """Mark a shard as clean (can be evicted).""" async with self._cache_lock: self._dirty_shards.discard(shard_idx) + self._evict_if_needed_locked() async def discard(self, shard_idx: ShardCacheKey) -> None: """Remove one shard from cache and dirty tracking.""" @@ -378,9 +422,8 @@ async def clear(self) -> None: self._shard_sizes.clear() self._actual_memory_usage = 0 - async def __contains__(self, shard_idx: ShardCacheKey) -> bool: - async with self._cache_lock: - return shard_idx in self._cache + def __contains__(self, shard_idx: ShardCacheKey) -> bool: + return shard_idx in self._cache @property def estimated_memory_usage(self) -> int: @@ -467,6 +510,11 @@ def __init__( self.array_indices: Dict[str, ArrayIndex] = {} self._primary_array_path: Optional[str] = None + # True only when the primary path was heuristically inferred for a + # legacy V1 root (never recorded on disk). Tracked separately from the + # path's truthiness because an inferred *root* primary is "", which is + # indistinguishable from "not inferred" under a plain truthiness check. + self._primary_inferred: bool = False self._default_chunks_per_shard: Optional[int] = None self._array_shape: Tuple[int, ...] = () @@ -724,6 +772,7 @@ async def _load_root_from_cid(self) -> None: manifest_version = self._root_obj.get("manifest_version") if manifest_version == SHARDED_ZARR_V1: self._load_v1_root() + await self._infer_v1_legacy_primary_array_path() elif manifest_version == SHARDED_ZARR_V2: self._load_v2_root() else: @@ -760,7 +809,105 @@ def _load_v1_root(self) -> None: shard_cids=chunk_info["shard_cids"], ) } - self._primary_array_path = "" + primary_array_path = chunk_info.get("primary_array_path", "") + self._primary_array_path = ( + self._normalize_array_path(primary_array_path) + if isinstance(primary_array_path, str) + else "" + ) + + async def _infer_v1_legacy_primary_array_path(self) -> None: + chunk_info = self._root_obj["chunks"] + # Only legacy roots missing the field may be inferred; recorded values win. + if "primary_array_path" in chunk_info: + return + + metadata = self._root_obj.get("metadata") + if not isinstance(metadata, dict): + return + + candidates: list[tuple[str, IPLDKind]] = [] + for key, metadata_cid in metadata.items(): + if not isinstance(key, str): + continue + array_path = self._array_path_from_metadata_key(key) + if array_path is None: + continue + normalized_path = self._normalize_array_path(array_path) + if normalized_path.rsplit("/", 1)[-1] in self._V1_COORDINATE_ARRAY_PREFIXES: + continue + + candidates.append((normalized_path, metadata_cid)) + + matching_paths: set[str] = set() + for batch_start in range(0, len(candidates), _V1_INFERENCE_CONCURRENCY): + batch = candidates[batch_start : batch_start + _V1_INFERENCE_CONCURRENCY] + metadata_results = await asyncio.gather( + *(self.cas.load(metadata_cid) for _, metadata_cid in batch), + return_exceptions=True, + ) + + for (normalized_path, _), metadata_result in zip(batch, metadata_results): + if isinstance(metadata_result, BaseException): + if isinstance(metadata_result, asyncio.CancelledError): + raise metadata_result + # A candidate we could not read might have been the true + # primary. Guessing from the survivors could rebind shard + # data under the wrong prefix, so abort the inference and + # keep the legacy default instead. + return + metadata_json = self._decode_metadata_json(metadata_result) + if metadata_json is None: + continue + declared_shape = metadata_json.get("shape") + if not ( + isinstance(declared_shape, (list, tuple)) + and tuple(declared_shape) == tuple(self._array_shape) + ): + continue + declared_chunk_shape = self._declared_chunk_shape(metadata_json) + if declared_chunk_shape != tuple(self._chunk_shape): + continue + matching_paths.add(normalized_path) + + # Once two different paths match, later metadata cannot make the + # inference unambiguous. Avoid issuing more remote CAS requests. + if len(matching_paths) > 1: + return + + if len(matching_paths) == 1: + # Inference only lands here when exactly one candidate matches, so + # the identified primary is correct. The flag keeps the + # (possibly empty-string) inferred primary exclusive so foreign + # chunk writes route to metadata instead of rebinding over it. + self._primary_array_path = next(iter(matching_paths)) + self._primary_inferred = True + if not self.read_only: + # Persist the unambiguous inference so a later reopen routes from + # a recorded primary rather than re-inferring. This is what makes + # routing deterministic: without it, adding a second + # same-geometry array would make the reopen inference ambiguous + # and misroute a metadata-stored chunk into the shard slot. + # Read-only opens cannot flush, so they rely on the flag alone. + self._root_obj["chunks"]["primary_array_path"] = ( + self._primary_array_path + ) + self._dirty_root = True + + @staticmethod + def _declared_chunk_shape(metadata_json: dict) -> Optional[tuple[int, ...]]: + """Extract the chunk shape a zarr v2/v3 array metadata document declares.""" + chunk_grid = metadata_json.get("chunk_grid") + if isinstance(chunk_grid, dict): + configuration = chunk_grid.get("configuration") + if isinstance(configuration, dict): + chunk_shape = configuration.get("chunk_shape") + if isinstance(chunk_shape, (list, tuple)): + return tuple(chunk_shape) + chunks = metadata_json.get("chunks") + if isinstance(chunks, (list, tuple)): + return tuple(chunks) + return None def _load_v2_root(self) -> None: metadata = self._root_obj.get("metadata") @@ -974,7 +1121,11 @@ async def _snapshot_shards_for_resize( for shard_idx, shard_cid_obj in enumerate(array_index.shard_cids): cache_key = self._cache_key(array_index.array_path, shard_idx) shard_lock = self._shard_locks[cache_key] - async with shard_lock: + # Pin across fetch+read: an over-budget cache whose older entries are + # all dirty or pinned would otherwise make the just-fetched clean + # shard the sole eviction candidate, evicting it before the snapshot + # read below and turning the load into a spurious RuntimeError. + async with shard_lock, self._shard_data_cache.pin(cache_key): shard_data = await self._shard_data_cache.get(cache_key) if shard_data is None and shard_cid_obj is not None: await self._fetch_and_cache_full_shard( @@ -1306,7 +1457,7 @@ async def _fetch_and_cache_full_shard( self, cache_key: ShardCacheKey, shard_idx: int, - shard_cid: str, + shard_cid: IPLDKind, expected_entries: int, max_retries: int = 3, retry_delay: float = 1.0, @@ -1346,7 +1497,7 @@ async def _load_sparse_shard_entry( self, cache_key: ShardCacheKey, shard_idx: int, - shard_cid: str, + shard_cid: IPLDKind, index_in_shard: int, expected_entries: int, ) -> Optional[CID]: @@ -1372,7 +1523,14 @@ def _parse_chunk_key(self, key: str) -> Optional[ChunkKey]: coord_part = key[marker_idx + len(chunk_marker) :] elif key.startswith("c/"): if self._manifest_version == SHARDED_ZARR_V1: - return None + recorded_path = self._root_obj.get("chunks", {}).get( + "primary_array_path" + ) + if not ( + isinstance(recorded_path, str) + and self._normalize_array_path(recorded_path) == "" + ): + return None array_path = "" coord_part = key[len("c/") :] else: @@ -1385,6 +1543,26 @@ def _parse_chunk_key(self, key: str) -> Optional[ChunkKey]: ) if actual_array_name in self._V1_COORDINATE_ARRAY_PREFIXES: return None + recorded_path = self._root_obj.get("chunks", {}).get("primary_array_path") + if isinstance(recorded_path, str): + primary_path_is_exclusive = True + effective_primary_path = recorded_path + elif self._primary_inferred or self._primary_array_path: + # An inferred primary is exclusive even when it is the root + # ("") — the flag, not the truthiness, decides. Without it a + # foreign same-rank named chunk would parse against the shard + # index and rebind the primary to itself. + primary_path_is_exclusive = True + effective_primary_path = self._primary_array_path or "" + else: + primary_path_is_exclusive = False + effective_primary_path = "" + if ( + primary_path_is_exclusive + and normalized_path + != self._normalize_array_path(effective_primary_path) + ): + return None parts = coord_part.split("/") try: @@ -1396,6 +1574,27 @@ def _parse_chunk_key(self, key: str) -> Optional[ChunkKey]: raise if self._manifest_version == SHARDED_ZARR_V1: + key_is_primary = ( + primary_path_is_exclusive + and normalized_path + == self._normalize_array_path(effective_primary_path) + ) + named_array_metadata = self._root_obj.get("metadata", {}) + if ( + normalized_path + and not key_is_primary + and len(coords) != len(self.array_indices[""].chunks_per_dim) + and ( + f"{normalized_path}/zarr.json" in named_array_metadata + or f"{normalized_path}/.zarray" in named_array_metadata + ) + ): + # A named array that registered its own metadata and whose + # rank differs from the primary geometry can never be a + # primary chunk: classify it as metadata instead of failing + # coordinate validation. Keys under the effective primary path + # still validate strictly so malformed primary keys fail loud. + return None self._validate_chunk_coords(coords, self.array_indices[""]) elif normalized_path in self.array_indices: self._validate_chunk_coords(coords, self.array_indices[normalized_path]) @@ -1540,7 +1739,7 @@ async def _get_legacy_metadata_chunk( return None req_offset, req_length, req_suffix = self._map_byte_request(byte_range) data = await self.cas.load( - str(metadata_cid_obj), + metadata_cid_obj, offset=req_offset, length=req_length, suffix=req_suffix, @@ -1549,6 +1748,17 @@ async def _get_legacy_metadata_chunk( async def _load_or_initialize_shard_cache( self, shard_idx: int, array_path: Optional[str] = None + ) -> List[Optional[CID]]: + """Return a shard after keeping it pinned throughout cache population.""" + array_index = self._array_index_for_path(array_path) + cache_key = self._cache_key(array_index.array_path, shard_idx) + async with self._shard_data_cache.pin(cache_key): + return await self._load_or_initialize_shard_cache_pinned( + shard_idx, array_path + ) + + async def _load_or_initialize_shard_cache_pinned( + self, shard_idx: int, array_path: Optional[str] = None ) -> List[Optional[CID]]: """ Load a shard into the cache or initialize an empty shard if it doesn't exist. @@ -1592,10 +1802,9 @@ async def _load_or_initialize_shard_cache( shard_cid_obj = array_index.shard_cids[shard_idx] if shard_cid_obj: self._pending_shard_loads[cache_key] = asyncio.Event() - shard_cid_str = str(shard_cid_obj) try: await self._fetch_and_cache_full_shard( - cache_key, shard_idx, shard_cid_str, array_index.chunks_per_shard + cache_key, shard_idx, shard_cid_obj, array_index.chunks_per_shard ) finally: pending_load = self._pending_shard_loads.pop(cache_key, None) @@ -1616,6 +1825,19 @@ async def _load_or_initialize_shard_cache( ) return result + @asynccontextmanager + async def _use_shard( + self, shard_idx: int, array_path: Optional[str] = None + ) -> AsyncIterator[List[Optional[CID]]]: + """Yield a pinned shard while serializing access to its contents.""" + array_index = self._array_index_for_path(array_path) + cache_key = self._cache_key(array_index.array_path, shard_idx) + async with self._shard_data_cache.pin(cache_key): + async with self._shard_locks[cache_key]: + yield await self._load_or_initialize_shard_cache( + shard_idx, array_index.array_path + ) + async def set_partial_values( self, key_start_values: Iterable[Tuple[str, int, BytesLike]] ) -> None: @@ -1659,6 +1881,7 @@ def with_read_only(self, read_only: bool = False) -> "ShardedZarrStore": clone.array_indices = self.array_indices clone._primary_array_path = self._primary_array_path + clone._primary_inferred = self._primary_inferred clone._default_chunks_per_shard = self._default_chunks_per_shard clone._array_shape = self._array_shape @@ -1671,6 +1894,29 @@ def with_read_only(self, read_only: bool = False) -> "ShardedZarrStore": clone._dirty_root = self._dirty_root clone._v2_pending_root_group_write = self._v2_pending_root_group_write + # A V1 primary that was inferred against a read-only open never reached + # the persistence branch in _infer_v1_legacy_primary_array_path (that + # branch is gated on `not self.read_only`), so it lives only in the + # in-memory `_primary_inferred` flag. Making a writable clone would carry + # that flag forward while leaving the shared root unrecorded: the first + # chunk write skips its seal-on-write (it is gated on + # `not self._primary_inferred`), so a same-geometry secondary array could + # flush with no recorded primary. On reopen inference would then be + # ambiguous and misroute the secondary's metadata chunk into the primary + # shard slot. Perform the deferred persistence the read-only open + # skipped, reproducing the state a writable open would have produced. + if ( + not read_only + and clone._manifest_version == SHARDED_ZARR_V1 + and clone._primary_inferred + and isinstance(clone._root_obj.get("chunks"), dict) + and "primary_array_path" not in clone._root_obj["chunks"] + ): + clone._root_obj["chunks"]["primary_array_path"] = ( + clone._primary_array_path or "" + ) + clone._dirty_root = True + zarr.abc.store.Store.__init__(clone, read_only=read_only) return clone @@ -1687,53 +1933,89 @@ async def _flush_unlocked(self) -> str: async with self._shard_data_cache._cache_lock: dirty_shards = list(self._shard_data_cache._dirty_shards) if dirty_shards: - for cache_key in sorted(dirty_shards, key=str): - shard_lock = self._shard_locks[cache_key] - async with shard_lock: - shard_data_list = await self._shard_data_cache.get(cache_key) - if shard_data_list is None: - raise RuntimeError( - f"Dirty shard {cache_key} not found in cache" - ) - - shard_data_bytes = dag_cbor.encode(cast(IPLDKind, shard_data_list)) - new_shard_cid_obj = await self.cas.save( - shard_data_bytes, - codec="dag-cbor", - ) - if not isinstance(new_shard_cid_obj, CID): # pragma: no cover - raise TypeError( - "ShardedZarrStore requires CAS.save to return CIDs." - ) - - if self._manifest_version == SHARDED_ZARR_V1: - if not isinstance(cache_key, int): # pragma: no cover - raise TypeError("v1 shard cache keys must be integers.") - shard_idx = int(cache_key) - if ( - self._root_obj["chunks"]["shard_cids"][shard_idx] - != new_shard_cid_obj - ): - self._root_obj["chunks"]["shard_cids"][shard_idx] = ( - new_shard_cid_obj + sorted_dirty_shards = sorted(dirty_shards, key=str) + next_shard_index = 0 + + async def flush_shards() -> None: + nonlocal next_shard_index + while next_shard_index < len(sorted_dirty_shards): + shard_index = next_shard_index + next_shard_index += 1 + cache_key = sorted_dirty_shards[shard_index] + async with self._shard_data_cache.pin(cache_key): + shard_lock = self._shard_locks[cache_key] + async with shard_lock: + shard_data_list = await self._shard_data_cache.get( + cache_key ) - self.array_indices[""].shard_cids[shard_idx] = ( - new_shard_cid_obj + if shard_data_list is None: + raise RuntimeError( + f"Dirty shard {cache_key} not found in cache" + ) + + shard_data_bytes = dag_cbor.encode( + cast(IPLDKind, shard_data_list) ) - self._dirty_root = True - else: - if isinstance(cache_key, int): # pragma: no cover - raise TypeError( - "v2 shard cache keys must include array paths." + new_shard_cid_obj = await self.cas.save( + shard_data_bytes, + codec="dag-cbor", ) - array_path, shard_idx = cache_key - array_index = self.array_indices[array_path] - if array_index.shard_cids[shard_idx] != new_shard_cid_obj: - array_index.shard_cids[shard_idx] = new_shard_cid_obj - self._dirty_root = True - self._sync_arrays_to_root() - - await self._shard_data_cache.mark_clean(cache_key) + if not isinstance( + new_shard_cid_obj, CID + ): # pragma: no cover + raise TypeError( + "ShardedZarrStore requires CAS.save to return CIDs." + ) + + if self._manifest_version == SHARDED_ZARR_V1: + if not isinstance(cache_key, int): # pragma: no cover + raise TypeError( + "v1 shard cache keys must be integers." + ) + shard_idx = int(cache_key) + if ( + self._root_obj["chunks"]["shard_cids"][shard_idx] + != new_shard_cid_obj + ): + self._root_obj["chunks"]["shard_cids"][ + shard_idx + ] = new_shard_cid_obj + self.array_indices[""].shard_cids[shard_idx] = ( + new_shard_cid_obj + ) + self._dirty_root = True + else: + if isinstance(cache_key, int): # pragma: no cover + raise TypeError( + "v2 shard cache keys must include array paths." + ) + array_path, shard_idx = cache_key + array_index = self.array_indices[array_path] + if ( + array_index.shard_cids[shard_idx] + != new_shard_cid_obj + ): + array_index.shard_cids[shard_idx] = ( + new_shard_cid_obj + ) + self._dirty_root = True + + await self._shard_data_cache.mark_clean(cache_key) + + flush_tasks = [ + asyncio.ensure_future(flush_shards()) + for _ in range(min(_FLUSH_CONCURRENCY, len(sorted_dirty_shards))) + ] + try: + await asyncio.gather(*flush_tasks) + except BaseException: + # No sibling task may outlive a failed flush: they would keep + # mutating store state (and using the CAS) without the write + # lock after the caller has already observed the failure. + for flush_task in flush_tasks: + flush_task.cancel() + await asyncio.gather(*flush_tasks, return_exceptions=True) + raise if self._dirty_root: self._root_obj["metadata"] = { @@ -1777,13 +2059,22 @@ async def get( metadata_cid_obj = self._root_obj["metadata"].get(lookup_key) if metadata_cid_obj is None: return None - if byte_range is not None: - raise ValueError( - "Byte range requests are not supported for metadata keys." - ) - data = self._metadata_read_cache.get(lookup_key) + data = ( + self._metadata_read_cache.get(lookup_key) + if byte_range is None + else None + ) if data is None: - data = await self.cas.load(str(metadata_cid_obj)) + req_offset, req_length, req_suffix = self._map_byte_request( + byte_range + ) + data = await self.cas.load( + metadata_cid_obj, + offset=req_offset, + length=req_length, + suffix=req_suffix, + ) + if byte_range is None: self._metadata_read_cache[lookup_key] = data hit = True return prototype.buffer.from_bytes(data) @@ -1818,7 +2109,7 @@ async def get( chunk_cid_obj = await self._load_sparse_shard_entry( cache_key, shard_idx, - str(array_index.shard_cids[shard_idx]), + cast(CID, array_index.shard_cids[shard_idx]), index_in_shard, array_index.chunks_per_shard, ) @@ -1835,11 +2126,9 @@ async def get( hit = legacy_buffer is not None return legacy_buffer - chunk_cid_str = str(chunk_cid_obj) - req_offset, req_length, req_suffix = self._map_byte_request(byte_range) data = await self.cas.load( - chunk_cid_str, + chunk_cid_obj, offset=req_offset, length=req_length, suffix=req_suffix, @@ -1883,7 +2172,9 @@ async def _set_unlocked(self, key: str, value: zarr.core.buffer.Buffer) -> None: try: data_cid_obj = await self.cas.save(raw_data_bytes, codec="raw") - await self._set_pointer(key, str(data_cid_obj), register_metadata=False) + if not isinstance(data_cid_obj, CID): + raise TypeError("ShardedZarrStore requires CAS.save to return CIDs.") + await self._set_pointer_cid(key, data_cid_obj, register_metadata=False) if parsed_chunk is None: self._metadata_read_cache[key] = raw_data_bytes except Exception as e: @@ -1899,6 +2190,13 @@ async def set_pointer(self, key: str, pointer: str) -> None: async def _set_pointer( self, key: str, pointer: str, *, register_metadata: bool + ) -> None: + await self._set_pointer_cid( + key, CID.decode(pointer), register_metadata=register_metadata + ) + + async def _set_pointer_cid( + self, key: str, pointer_cid_obj: CID, *, register_metadata: bool ) -> None: try: parsed_chunk = self._parse_chunk_key(key) @@ -1906,11 +2204,9 @@ async def _set_pointer( if self._manifest_version != SHARDED_ZARR_V2: raise return None - pointer_cid_obj = CID.decode(pointer) - if parsed_chunk is None: if register_metadata and self._manifest_version == SHARDED_ZARR_V2: - raw_metadata = await self.cas.load(str(pointer_cid_obj)) + raw_metadata = await self.cas.load(pointer_cid_obj) stripped_metadata = self._strip_v2_root_consolidated_metadata( key, raw_metadata ) @@ -1927,7 +2223,7 @@ async def _set_pointer( register_metadata and self._array_path_from_metadata_key(key) is not None ): - raw_metadata = await self.cas.load(str(pointer_cid_obj)) + raw_metadata = await self.cas.load(pointer_cid_obj) await self._register_array_metadata_from_bytes(key, raw_metadata) if register_metadata: await self._ensure_v2_parent_group_metadata(key) @@ -1945,15 +2241,28 @@ async def _set_pointer( ) cache_key = self._cache_key(array_index.array_path, shard_idx) - shard_lock = self._shard_locks[cache_key] - async with shard_lock: - target_shard_list = await self._load_or_initialize_shard_cache( - shard_idx, array_index.array_path + async with self._use_shard(shard_idx, array_index.array_path): + await self._shard_data_cache.update_entry( + cache_key, index_in_shard, pointer_cid_obj ) + # A legacy root may also carry this chunk key in the metadata + # mapping; drop it so the superseded CID is not pinned forever and + # cannot resurface through the legacy fallback once the shard slot + # empties again. + if self._root_obj["metadata"].pop(key, None) is not None: + self._metadata_read_cache.pop(key, None) + self._dirty_root = True - if target_shard_list[index_in_shard] != pointer_cid_obj: - target_shard_list[index_in_shard] = pointer_cid_obj - await self._shard_data_cache.mark_dirty(cache_key) + if self._manifest_version == SHARDED_ZARR_V1 and not self._primary_inferred: + # A genuinely-unrecorded root's first chunk write establishes and + # persists its primary. An *inferred* primary is never sealed here: + # inference is in-memory only, and the exclusivity flag guarantees + # only true-primary chunks reach this point anyway. + chunk_info = self._root_obj["chunks"] + if "primary_array_path" not in chunk_info: + self._primary_array_path = parsed_chunk.array_path + chunk_info["primary_array_path"] = parsed_chunk.array_path + self._dirty_root = True return None async def exists(self, key: str) -> bool: @@ -1972,12 +2281,12 @@ async def exists(self, key: str) -> bool: shard_idx, index_in_shard = self._get_shard_info_for_index( linear_chunk_index, array_index ) - target_shard_list = await self._load_or_initialize_shard_cache( + async with self._use_shard( shard_idx, array_index.array_path - ) - return target_shard_list[ - index_in_shard - ] is not None or lookup_key in self._root_obj.get("metadata", {}) + ) as target_shard_list: + return target_shard_list[ + index_in_shard + ] is not None or lookup_key in self._root_obj.get("metadata", {}) except (ValueError, IndexError, KeyError): return False @@ -2029,15 +2338,9 @@ async def _delete_unlocked(self, key: str) -> None: ) cache_key = self._cache_key(array_index.array_path, shard_idx) - shard_lock = self._shard_locks[cache_key] - async with shard_lock: - target_shard_list = await self._load_or_initialize_shard_cache( - shard_idx, array_index.array_path - ) - if target_shard_list[index_in_shard] is not None: - target_shard_list[index_in_shard] = None - await self._shard_data_cache.mark_dirty(cache_key) - elif self._root_obj["metadata"].pop(key, None) is not None: + async with self._use_shard(shard_idx, array_index.array_path): + await self._shard_data_cache.update_entry(cache_key, index_in_shard, None) + if self._root_obj["metadata"].pop(key, None) is not None: self._metadata_read_cache.pop(key, None) self._dirty_root = True @@ -2148,13 +2451,17 @@ async def list(self) -> AsyncIterator[str]: yielded.add(key) yield key - if self._manifest_version == SHARDED_ZARR_V2: - async for chunk_key in self._iter_chunk_keys(): - if chunk_key not in yielded: - yield chunk_key + async for chunk_key in self._iter_chunk_keys(): + if chunk_key not in yielded: + yield chunk_key async def _iter_chunk_keys(self) -> AsyncIterator[str]: for array_path, array_index in self.array_indices.items(): + listed_array_path = ( + self._primary_array_path + if self._manifest_version == SHARDED_ZARR_V1 + else array_path + ) for shard_idx in range(array_index.num_shards): cache_key = self._cache_key(array_path, shard_idx) shard_data = await self._shard_data_cache.get(cache_key) @@ -2176,7 +2483,22 @@ async def _iter_chunk_keys(self) -> AsyncIterator[str]: coords = self._coords_from_linear_index( linear_index, array_index.chunks_per_dim ) - yield self._format_chunk_key(array_path, coords) + chunk_key = self._format_chunk_key(listed_array_path or "", coords) + if ( + self._manifest_version == SHARDED_ZARR_V1 + and not listed_array_path + ): + recorded_path = self._root_obj.get("chunks", {}).get( + "primary_array_path" + ) + if not ( + isinstance(recorded_path, str) + and self._normalize_array_path(recorded_path) == "" + ): + # Unrecorded V1 roots distinguish shard chunks + # ("/c/...") from legacy metadata keys ("c/..."). + chunk_key = f"/{chunk_key}" + yield chunk_key async def list_prefix(self, prefix: str) -> AsyncIterator[str]: async for key in self.list(): @@ -2186,6 +2508,10 @@ async def list_prefix(self, prefix: str) -> AsyncIterator[str]: def _list_dir_candidate_keys(self) -> Set[str]: keys = set(self._root_obj.get("metadata", {})) if self._manifest_version != SHARDED_ZARR_V2: + chunk_prefix = ( + "c" if not self._primary_array_path else f"{self._primary_array_path}/c" + ) + keys.add(chunk_prefix) return keys for array_path in self.array_indices: @@ -2195,11 +2521,20 @@ def _list_dir_candidate_keys(self) -> Set[str]: keys.add("c" if array_path == "" else f"{array_path}/c") return keys - def _is_v2_chunk_listing_prefix(self, normalized_prefix: str) -> bool: - if self._manifest_version != SHARDED_ZARR_V2: - return False - for array_path in self.array_indices: - chunk_prefix = "c" if array_path == "" else f"{array_path}/c" + def _chunk_listing_prefixes(self) -> Iterator[str]: + """The chunk-directory prefixes ``_iter_chunk_keys`` emits keys under.""" + if self._manifest_version == SHARDED_ZARR_V2: + for array_path in self.array_indices: + yield "c" if array_path == "" else f"{array_path}/c" + else: + # V1 emits a single primary chunk tree, rooted at "c" or + # "/c" to match the recorded/inferred primary path. + yield ( + "c" if not self._primary_array_path else f"{self._primary_array_path}/c" + ) + + def _is_chunk_listing_prefix(self, normalized_prefix: str) -> bool: + for chunk_prefix in self._chunk_listing_prefixes(): if normalized_prefix == chunk_prefix or normalized_prefix.startswith( f"{chunk_prefix}/" ): @@ -2292,14 +2627,10 @@ async def _graft_store_unlocked( ) cache_key = self._cache_key(target_index.array_path, global_shard_idx) - shard_lock = self._shard_locks[cache_key] - async with shard_lock: - target_shard_list = await self._load_or_initialize_shard_cache( - global_shard_idx, target_index.array_path + async with self._use_shard(global_shard_idx, target_index.array_path): + await self._shard_data_cache.update_entry( + cache_key, index_in_global_shard, pointer_cid_obj ) - if target_shard_list[index_in_global_shard] != pointer_cid_obj: - target_shard_list[index_in_global_shard] = pointer_cid_obj - await self._shard_data_cache.mark_dirty(cache_key) async def resize_store( self, new_shape: Tuple[int, ...], *, array_path: Optional[str] = None @@ -2459,11 +2790,16 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]: effective_prefix = self._v2_effective_list_dir_prefix(normalized_prefix) match_prefix = f"{effective_prefix}/" if effective_prefix else "" - if self._is_v2_chunk_listing_prefix(effective_prefix): + if self._is_chunk_listing_prefix(effective_prefix): async for key in self._iter_chunk_keys(): - if not key.startswith(match_prefix): + # Unrecorded V1 roots emit the primary shard tree with a + # leading slash ("/c/...") to distinguish it from legacy + # metadata keys, but list_dir prefixes are slash-stripped. + # Normalize the emitted key so list_dir("c") matches "/c/...". + normalized_key = key[1:] if key.startswith("/") else key + if not normalized_key.startswith(match_prefix): continue - suffix = key[len(match_prefix) :] + suffix = normalized_key[len(match_prefix) :] first_component = suffix.split("/", 1)[0] if first_component not in seen: seen.add(first_component) diff --git a/py_hamt/store_httpx.py b/py_hamt/store_httpx.py index 66b39ba..ae0e950 100644 --- a/py_hamt/store_httpx.py +++ b/py_hamt/store_httpx.py @@ -1,8 +1,13 @@ import asyncio +import logging import random import re +import threading +import warnings from abc import ABC, abstractmethod -from typing import Any, Dict, Literal, Optional, Tuple, cast +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from typing import Any, Callable, Dict, Literal, Optional, Tuple, cast import httpx from dag_cbor.ipld import IPLDKind @@ -11,6 +16,158 @@ from . import instrumentation +logger = logging.getLogger(__name__) + +_RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504}) + +# Ceiling on server-directed waits so a broken or hostile gateway cannot make +# a request sleep unbounded (e.g. ``Retry-After: inf`` or a far-future date). +_MAX_RETRY_AFTER_SECONDS = 300.0 + + +def _retry_delay( + initial_delay: float, + backoff_factor: float, + retry_number: int, + response: Optional[httpx.Response] = None, +) -> float: + """Return a valid ``Retry-After`` value, otherwise a jittered backoff.""" + backoff_delay = initial_delay * (backoff_factor ** (retry_number - 1)) + retry_after = response.headers.get("Retry-After") if response is not None else None + if retry_after is not None: + try: + retry_after_seconds = float(retry_after) + except ValueError: + try: + retry_at = parsedate_to_datetime(retry_after) + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + retry_after_seconds = ( + retry_at - datetime.now(timezone.utc) + ).total_seconds() + except (TypeError, ValueError, OverflowError): + retry_after_seconds = -1 + + if retry_after_seconds >= 0: + return min(retry_after_seconds, _MAX_RETRY_AFTER_SECONDS) + + jitter = backoff_delay * 0.1 * (random.random() - 0.5) + return backoff_delay + jitter + + +def _slice_requested_range( + data: bytes, + offset: Optional[int], + length: Optional[int], + suffix: Optional[int], +) -> bytes: + """Apply content-store range arguments to a complete object body.""" + if offset is not None: + if length is not None: + return data[offset : offset + length] + return data[offset:] + if suffix is not None: + if suffix == 0: + return b"" + return data[-suffix:] + return data + + +def _range_not_satisfiable_is_empty( + response: httpx.Response, offset: Optional[int], suffix: Optional[int] +) -> bool: + """Whether a ``416`` response describes a read Python slicing treats as empty. + + A spec-compliant gateway rejects an unsatisfiable range with ``416 Range Not + Satisfiable`` and a ``Content-Range: bytes */N`` header giving the object + size ``N``. Python slice semantics (and ``InMemoryCAS``) yield ``b""`` for + those same reads, so ``KuboCAS`` matches by treating them as empty instead of + surfacing the error: + + * an ``offset`` read is empty when it starts at or past EOF (``offset >= N``); + * a ``suffix`` read is unsatisfiable only against a zero-length object + (``N == 0``); ``data[-suffix:]`` on an empty object is likewise ``b""``. + (For a non-empty object a suffix range is always satisfiable, so a ``416`` + there is a genuine error and is surfaced.) + """ + content_range = response.headers.get("Content-Range", "") + match = re.fullmatch(r"bytes \*/(\d+)", content_range.strip()) + if match is None: + return False + total = int(match.group(1)) + if suffix is not None: + # A suffix range only fails against a zero-length object. + return total == 0 + return offset is not None and offset >= total + + +def _validate_partial_content( + response: httpx.Response, + offset: Optional[int], + length: Optional[int], + suffix: Optional[int], + body_len: int, +) -> None: + """Reject a ``206`` whose byte window is missing or inconsistent. + + ``raise_for_status`` accepts any 2xx, so a gateway can answer a Range request + with a malformed ``206`` -- an absent, unparseable, or ``*``-total + ``Content-Range``, a declared window that disagrees with the body length, or + a window that does not match what was requested -- and silently hand back the + wrong bytes. We recompute the exact window the request maps to (the object + size ``N`` is always known for content-addressed reads) and raise an + ``httpx.HTTPStatusError`` on any mismatch so a corrupt partial read fails + loudly rather than corrupting the caller's data. + """ + content_range = response.headers.get("Content-Range", "") + match = re.fullmatch(r"bytes (\d+)-(\d+)/(\d+)", content_range.strip()) + if match is None: + raise httpx.HTTPStatusError( + f"malformed 206 Content-Range {content_range!r}", + request=response.request, + response=response, + ) + start, end, total = (int(group) for group in match.groups()) + # The declared inclusive window must match the number of bytes delivered. + if end - start + 1 != body_len: + raise httpx.HTTPStatusError( + f"206 Content-Range {content_range!r} declares " + f"{end - start + 1} bytes but body is {body_len}", + request=response.request, + response=response, + ) + # Recompute the window the request maps to and demand an exact match, so a + # gateway cannot return a shifted or wrong-sized slice of the object. + if offset is not None: + expected_start = offset + expected_len = total - offset if length is None else min(length, total - offset) + else: + expected_start = max(total - cast(int, suffix), 0) + expected_len = min(cast(int, suffix), total) + if start != expected_start or body_len != expected_len: + raise httpx.HTTPStatusError( + f"206 Content-Range {content_range!r} (start={start}, {body_len} bytes) " + f"does not match the requested window " + f"(start={expected_start}, {expected_len} bytes)", + request=response.request, + response=response, + ) + + +# Upper bound on how long aclose() waits for a cross-loop client close that was +# scheduled onto a *running* owner loop. Bounds the window where that loop stops +# between the is_running() check and the coroutine executing, which would +# otherwise leave the wrapped future pending forever. Module-level so tests can +# shrink it; a healthy running loop closes near-instantly, well under this. +_CROSS_LOOP_ACLOSE_TIMEOUT_S = 30.0 + + +def _close_client_on_stopped_loop( + owner_loop: asyncio.AbstractEventLoop, client: httpx.AsyncClient +) -> None: + """Run client cleanup on its stopped but still usable owner loop.""" + owner_loop.run_until_complete(client.aclose()) + class ContentAddressedStore(ABC): """ @@ -85,6 +242,10 @@ async def load( suffix: Optional[int] = None, ) -> bytes: """ + Retrieve all or part of an object using Python slice semantics. + + A zero ``length`` or ``suffix`` returns an empty byte string. + `ContentAddressedStore` allows any IPLD scalar key. For the in-memory backend we *require* a `bytes` hash; anything else is rejected at run time. In OO type-checking, a subclass may widen (make more general) argument types, @@ -93,6 +254,9 @@ async def load( This is why we use `cast` here, to tell mypy that we know what we are doing. h/t https://stackoverflow.com/questions/75209249/overriding-a-method-mypy-throws-an-incompatible-with-super-type-error-when-ch """ + if (offset is not None and length == 0) or (offset is None and suffix == 0): + return b"" + key = cast(bytes, id) if not isinstance(key, (bytes, bytearray)): # defensive guard raise TypeError( @@ -104,17 +268,7 @@ async def load( except KeyError as exc: raise KeyError("Object not found in in-memory store") from exc - if offset is not None: - start = offset - if length is not None: - end = start + length - return data[start:end] - else: - return data[start:] - elif suffix is not None: # If only length is given, assume start from 0 - return data[-suffix:] - else: # Full load - return data + return _slice_requested_range(data, offset, length, suffix) class KuboCAS(ContentAddressedStore): @@ -131,12 +285,15 @@ class KuboCAS(ContentAddressedStore): ### Authentication / custom headers You have two options: - 1. **Bring your own `httpx.AsyncClient`** - Pass it via `client=...` — any default headers or auth - configured on that client are reused for **every** request. + 1. **Bring your own `httpx.AsyncClient` or client factory** + Pass a client via `client=...` for use on one event loop, or pass + `client_factory=...` to build a fully configured client for each event + loop. Reusing a supplied client from a later loop emits a warning and + falls back to an internal client that preserves only headers, auth, + timeout, redirect policy, and event hooks. 2. **Let `KuboCAS` build the client** but pass `headers=` *and*/or `auth=` kwargs; they are forwarded to the - internally–created `AsyncClient`. + internally-created `AsyncClient`. ```python import httpx @@ -146,6 +303,7 @@ class KuboCAS(ContentAddressedStore): client = httpx.AsyncClient( headers={"Authorization": "Bearer "}, auth=("user", "pass"), + follow_redirects=True, ) cas = KuboCAS(client=client) @@ -159,13 +317,22 @@ class KuboCAS(ContentAddressedStore): ### Parameters - **hasher** (str): multihash name (defaults to *blake3*). - **client** (`httpx.AsyncClient | None`): reuse an existing - client; if *None* KuboCAS will create one lazily. + client and its configured timeout and redirect policy. User-supplied + clients should set ``follow_redirects=True`` when gateways may redirect. + If *None*, KuboCAS will create one lazily with a 60-second timeout and + redirect following and HTTP/2 enabled. Plaintext endpoints continue to + use HTTP/1.1 because HTTP/2 negotiation requires TLS/ALPN. + - **client_factory** (`Callable[[], httpx.AsyncClient] | None`): create a + separate, fully configured client for each event loop. KuboCAS owns and + closes clients returned by the factory. Mutually exclusive with + **client**. - **headers** (dict[str, str] | None): default headers for the internally-created client. - **auth** (`tuple[str, str] | None`): authentication tuple (username, password) for the internally-created client. - **rpc_base_url / gateway_base_url** (str | None): override daemon - endpoints (defaults match the local daemon ports). + endpoints (defaults match the local daemon ports). Gateway URLs may end + with `/ipfs` and may include a trailing slash. - **chunker** (str): chunking algorithm specification for Kubo's `add` RPC. Accepted formats are `"size-"`, `"rabin"`, or `"rabin---"`. @@ -188,6 +355,7 @@ def __init__( gateway_base_url: str | None = None, concurrency: int = 32, *, + client_factory: Optional[Callable[[], httpx.AsyncClient]] = None, headers: dict[str, str] | None = None, auth: Tuple[str, str] | None = None, pin_on_add: bool = False, @@ -203,6 +371,17 @@ def __init__( If `client` is not provided, it will be automatically initialized. It is the responsibility of the user to close this at an appropriate time, using `await cas.aclose()` as a class instance cannot know when it will no longer be in use, unless explicitly told to do so. + A supplied client is associated with the running event loop lazily on + first use, so constructing ``KuboCAS`` does not require an async + context. On a later event loop, KuboCAS warns and uses an internally + created fallback that preserves only the supplied client's headers, + auth, timeout, redirect policy, and event hooks. Pass + ``client_factory`` instead when every event loop needs the client's + full configuration. Factory clients are owned and closed by KuboCAS. + Clients created internally by ``KuboCAS`` use a 60-second timeout, + follow redirects, and negotiate HTTP/2 for HTTPS endpoints that + support it. + If you are using the `KuboCAS` instance in an `async with` block, it will automatically close the client when the block is exited which is what we suggest below: ```python async with httpx.AsyncClient() as client, KuboCAS( @@ -232,11 +411,25 @@ def __init__( These are the first part of the url, defaults that refer to the default that kubo launches with on a local machine are provided. """ + if client is not None and client_factory is not None: + raise ValueError("client and client_factory are mutually exclusive") + if client_factory is not None and (headers is not None or auth is not None): + raise ValueError( + "client_factory is mutually exclusive with headers/auth; " + "configure them on the clients the factory builds" + ) + self._owns_client: bool = False self._closed: bool = True self._client_per_loop: Dict[asyncio.AbstractEventLoop, httpx.AsyncClient] = {} - self._default_headers = headers - self._default_auth = auth + self._internally_created_clients: set[httpx.AsyncClient] = set() + # Serializes first-use client binding so concurrent event loops on + # different threads cannot both consume ``_supplied_client`` and bind + # one httpx.AsyncClient to two loops. + self._first_use_lock: threading.Lock = threading.Lock() + self._semaphore_per_loop: Dict[ + asyncio.AbstractEventLoop, asyncio.Semaphore + ] = {} # Now, perform validation that might raise an exception chunker_pattern = r"(?:size-[1-9]\d*|rabin(?:-[1-9]\d*-[1-9]\d*-[1-9]\d*)?)" @@ -252,14 +445,10 @@ def __init__( if gateway_base_url is None: gateway_base_url = KuboCAS.KUBO_DEFAULT_LOCAL_GATEWAY_BASE_URL - if "/ipfs/" in gateway_base_url: - gateway_base_url = gateway_base_url.split("/ipfs/")[0] - - # Standard gateway URL construction with proper path handling - if gateway_base_url.endswith("/"): - gateway_base_url = f"{gateway_base_url}ipfs/" - else: - gateway_base_url = f"{gateway_base_url}/ipfs/" + gateway_base_url = gateway_base_url.rstrip("/") + if not gateway_base_url.endswith("/ipfs"): + gateway_base_url = f"{gateway_base_url}/ipfs" + gateway_base_url = f"{gateway_base_url}/" pin_string: str = "true" if pin_on_add else "false" self.rpc_url: str = f"{rpc_base_url}/api/v0/add?hash={self.hasher}&chunker={self.chunker}&pin={pin_string}" @@ -268,19 +457,40 @@ def __init__( """@private""" if client is not None: - # A client was supplied by the user. We don't own it. + # Bind the user-supplied client lazily on first async use. self._owns_client = False - self._client_per_loop = {asyncio.get_running_loop(): client} + self._supplied_client: httpx.AsyncClient | None = client + self._user_client: httpx.AsyncClient | None = client + self._default_headers: httpx.Headers | dict[str, str] | None = ( + httpx.Headers(client.headers) + ) + self._default_auth: httpx.Auth | Tuple[str, str] | None = client.auth + self._default_timeout: httpx.Timeout | float = client.timeout + self._default_limits = self._copy_client_limits(client) + self._default_follow_redirects: bool = client.follow_redirects + # Snapshot the hooks like the headers above: later mutations of the + # supplied client must not leak into fallback clients. + self._default_event_hooks: dict[str, list[Callable[..., Any]]] | None = { + event: list(hooks) for event, hooks in client.event_hooks.items() + } else: # No client supplied. We will own any clients we create. self._owns_client = True - self._client_per_loop = {} - - # store for later use by _loop_client() - self._default_headers = headers - self._default_auth = auth + self._supplied_client = None + self._user_client = None + self._default_headers = headers + self._default_auth = auth + self._default_timeout = 60.0 + self._default_limits = httpx.Limits( + max_connections=64, max_keepalive_connections=32 + ) + self._default_follow_redirects = True + self._default_event_hooks = None + self._client_factory: Optional[Callable[[], httpx.AsyncClient]] = client_factory - self._sem: asyncio.Semaphore = asyncio.Semaphore(concurrency) + if concurrency <= 0: + raise ValueError("concurrency must be a positive integer") + self._concurrency: int = concurrency self._closed = False # Validate retry parameters @@ -295,16 +505,55 @@ def __init__( self.initial_delay = initial_delay self.backoff_factor = backoff_factor + @staticmethod + def _copy_client_limits(client: httpx.AsyncClient) -> httpx.Limits: + """Copy connection limits from a standard HTTPX async transport. + + HTTPX does not expose client limits publicly, so custom transports fall + back to the limits KuboCAS uses for its own clients. + """ + transport: Any = client._transport + pool: Any = getattr(transport, "_pool", None) + return httpx.Limits( + max_connections=getattr(pool, "_max_connections", 64), + max_keepalive_connections=getattr(pool, "_max_keepalive_connections", 32), + keepalive_expiry=getattr(pool, "_keepalive_expiry", 5.0), + ) + # --------------------------------------------------------------------- # # helper: get or create the client bound to the current running loop # # --------------------------------------------------------------------- # + def _loop_semaphore(self) -> asyncio.Semaphore: + """Get or create the concurrency semaphore for the running event loop. + + Semaphores cannot be shared safely across event loops once contended, + so their lifecycle mirrors the per-loop HTTP clients. + """ + if self._closed: + if not self._owns_client: + raise RuntimeError("KuboCAS is closed; create a new instance") + self._closed = False + self._client_per_loop = {} + self._internally_created_clients = set() + self._semaphore_per_loop = {} + + loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + try: + return self._semaphore_per_loop[loop] + except KeyError: + semaphore = asyncio.Semaphore(self._concurrency) + self._semaphore_per_loop[loop] = semaphore + return semaphore + def _loop_client(self) -> httpx.AsyncClient: """Get or create a client for the current event loop. + A user-supplied client is bound to the first loop that requests it. If the instance was previously closed but owns its clients, a fresh - client mapping is lazily created on demand. Users that supplied their + client mapping is lazily created on demand. Users that supplied their own ``httpx.AsyncClient`` still receive an error when the instance has - been closed, as we cannot safely recreate their client. + been closed, as we cannot safely recreate their client. Internally + created clients enable HTTP/2 negotiation for HTTPS endpoints. """ if self._closed: if not self._owns_client: @@ -313,43 +562,122 @@ def _loop_client(self) -> httpx.AsyncClient: # state so that new clients can be created lazily. self._closed = False self._client_per_loop = {} + self._semaphore_per_loop = {} loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() try: return self._client_per_loop[loop] except KeyError: - # Create a new client - client = httpx.AsyncClient( - timeout=60.0, - headers=self._default_headers, - auth=self._default_auth, - limits=httpx.Limits(max_connections=64, max_keepalive_connections=32), - # Uncomment when they finally support Robust HTTP/2 GOAWAY responses - # http2=True, - ) - self._client_per_loop[loop] = client - return client + # First use on this loop. Hold the lock across supplied-client + # consumption, client creation, and the per-loop assignment so two + # loops racing on different threads cannot bind the same client. + with self._first_use_lock: + if self._supplied_client is not None: + client = self._supplied_client + self._supplied_client = None + elif self._client_factory is not None: + client = self._client_factory() + self._internally_created_clients.add(client) + else: + if self._user_client is not None: + warnings.warn( + "A user-supplied httpx.AsyncClient cannot be reused " + "across event loops; falling back to an internally " + "created client that preserves only headers, auth, " + "timeout, limits, redirect policy, and event hooks. " + "Pass client_factory to preserve full configuration.", + RuntimeWarning, + stacklevel=2, + ) + client = httpx.AsyncClient( + timeout=self._default_timeout, + headers=self._default_headers, + auth=self._default_auth, + limits=self._default_limits, + follow_redirects=self._default_follow_redirects, + event_hooks=self._default_event_hooks, + http2=True, + ) + self._internally_created_clients.add(client) + self._client_per_loop[loop] = client + return client # --------------------------------------------------------------------- # # graceful shutdown: close **all** clients we own # # --------------------------------------------------------------------- # async def aclose(self) -> None: """ - Closes all internally-created clients. Must be called from an async context. + Close every internally-created client, leaving a supplied client open. + + Must be called from an async context. + + For clients owned by closed loops with stock async-only transports, + cleanup degenerates to a warning. The OS-level socket is shut down + with a FIN, but its local file descriptor is released at garbage + collection. Callers that require deterministic release should call + ``aclose()`` on the owning loop before it exits. """ - if self._owns_client is False: # external client → caller closes - return + try: + current_loop: asyncio.AbstractEventLoop | None = asyncio.get_running_loop() + except RuntimeError: + current_loop = None + + for owner_loop, client in list(self._client_per_loop.items()): + if client not in self._internally_created_clients: + continue + + try: + if owner_loop is current_loop: + await client.aclose() + continue + + if not owner_loop.is_closed(): + if owner_loop.is_running(): + close_future = asyncio.run_coroutine_threadsafe( + client.aclose(), owner_loop + ) + try: + await asyncio.wait_for( + asyncio.wrap_future(close_future), + timeout=_CROSS_LOOP_ACLOSE_TIMEOUT_S, + ) + except TimeoutError: + # The owner loop stopped (or stalled) after + # is_running() succeeded, so the scheduled close can + # never complete. Cancel it and fall through to the + # synchronous transport shutdown below. + close_future.cancel() + else: + continue + else: + await asyncio.to_thread( + _close_client_on_stopped_loop, owner_loop, client + ) + continue - # This method is async, so we can reliably await the async close method. - # The complex sync/async logic is handled by __del__. - for client in list(self._client_per_loop.values()): - if not client.is_closed: + # AsyncClient marks itself closed before awaiting its transport. + # A dead owner loop therefore needs the transport's sync fallback. + transport: Any = client._transport + close_transport = getattr(transport, "close", None) + if close_transport is None: + await client.aclose() + continue + + close_transport() try: await client.aclose() except Exception: - pass # best-effort cleanup + pass # The transport was already closed synchronously. + except Exception as exc: + warnings.warn( + f"Failed to close an internally created HTTP client: {exc}", + RuntimeWarning, + stacklevel=2, + ) self._client_per_loop.clear() + self._internally_created_clients.clear() + self._semaphore_per_loop.clear() self._closed = True # At this point, _client_per_loop should be empty or only contain @@ -365,7 +693,10 @@ def __del__(self) -> None: if not hasattr(self, "_owns_client") or not hasattr(self, "_closed"): return - if not self._owns_client or self._closed: + if ( + not self._owns_client + and not getattr(self, "_internally_created_clients", set()) + ) or self._closed: return # Attempt proper cleanup if possible @@ -378,6 +709,7 @@ def __del__(self) -> None: # We can't await client.aclose() without a loop, # so just clear the references self._client_per_loop.clear() + self._semaphore_per_loop.clear() self._closed = True return @@ -399,50 +731,58 @@ def __del__(self) -> None: # If all else fails, just clear references if hasattr(self, "_client_per_loop"): self._client_per_loop.clear() + self._semaphore_per_loop.clear() self._closed = True # --------------------------------------------------------------------- # - # save() – now uses the per-loop client # + # save() - now uses the per-loop client # # --------------------------------------------------------------------- # async def save(self, data: bytes, codec: ContentAddressedStore.CodecInput) -> CID: - async with self._sem: - files = {"file": data} - client = self._loop_client() - retry_count = 0 - - while retry_count <= self.max_retries: - try: - response = await client.post( - self.rpc_url, files=files, timeout=60.0 - ) - response.raise_for_status() - cid_str: str = response.json()["Hash"] - cid: CID = CID.decode(cid_str) - if cid.codec.code != self.DAG_PB_MARKER: - cid = cid.set(codec=codec) - return cid + """Add data to Kubo and return its CID. - except (httpx.TimeoutException, httpx.RequestError) as e: - retry_count += 1 - if retry_count > self.max_retries: - raise httpx.TimeoutException( - f"Failed to save data after {self.max_retries} retries: {str(e)}", - request=e.request - if isinstance(e, httpx.RequestError) - else None, - ) - - # Calculate backoff delay - delay = self.initial_delay * ( - self.backoff_factor ** (retry_count - 1) - ) - # Add some jitter to prevent thundering herd - jitter = delay * 0.1 * (random.random() - 0.5) - await asyncio.sleep(delay + jitter) + Transient request failures and gateway statuses are retried. Retrying + the ``/api/v0/add`` POST is safe because the uploaded content is + content-addressed, making repeated additions idempotent. Concurrency + slots are held per HTTP attempt and released during retry backoff. + """ + files = {"file": data} + client = self._loop_client() + semaphore = self._loop_semaphore() + retry_count = 0 + + while retry_count <= self.max_retries: + try: + async with semaphore: + response = await client.post(self.rpc_url, files=files) + response.raise_for_status() + cid_str: str = response.json()["Hash"] + cid: CID = CID.decode(cid_str) + if cid.codec.code != self.DAG_PB_MARKER: + cid = cid.set(codec=codec) + return cid + + except httpx.RequestError: + if retry_count >= self.max_retries: + raise + retry_count += 1 + await asyncio.sleep( + _retry_delay(self.initial_delay, self.backoff_factor, retry_count) + ) - except httpx.HTTPStatusError: - # Re-raise non-timeout HTTP errors immediately + except httpx.HTTPStatusError as error: + if error.response.status_code not in _RETRYABLE_STATUS_CODES: + raise + if retry_count >= self.max_retries: raise + retry_count += 1 + await asyncio.sleep( + _retry_delay( + self.initial_delay, + self.backoff_factor, + retry_count, + error.response, + ) + ) raise RuntimeError("Exited the retry loop unexpectedly.") # pragma: no cover async def load( @@ -452,7 +792,18 @@ async def load( length: Optional[int] = None, suffix: Optional[int] = None, ) -> bytes: - """Load data from a CID using the IPFS gateway with optional Range requests.""" + """Load all or part of a CID using the IPFS gateway. + + Gateways that ignore a Range header and return a complete ``200`` body + are handled by applying the requested byte window locally. Transient + request failures, rate limits, and gateway server errors are retried; + other HTTP errors fail immediately. Zero-length and zero-suffix reads + return immediately without a gateway request. Concurrency slots are + held per HTTP attempt and released during retry backoff. + """ + if (offset is not None and length == 0) or (offset is None and suffix == 0): + return b"" + cid = cast(CID, id) url: str = f"{self.gateway_base_url + str(cid)}" headers: Dict[str, str] = {} @@ -476,45 +827,85 @@ async def load( final_status = "ok" final_retry_count = 0 try: - async with self._sem: # Throttle gateway - client = self._loop_client() - retry_count = 0 - - while retry_count <= self.max_retries: - try: - response = await client.get( - url, headers=headers or None, timeout=60.0 - ) - response.raise_for_status() - content = response.content - response_bytes = len(content) + client = self._loop_client() + semaphore = self._loop_semaphore() + retry_count = 0 + + while retry_count <= self.max_retries: + try: + async with semaphore: # Throttle each gateway attempt + response = await client.get(url, headers=headers or None) + # An unsatisfiable range is answered with 416 by a compliant + # gateway; return b"" to match Python-slice semantics (and + # InMemoryCAS) instead of raising. + if ( + response.status_code + == httpx.codes.REQUESTED_RANGE_NOT_SATISFIABLE + and _range_not_satisfiable_is_empty(response, offset, suffix) + ): final_retry_count = retry_count - return content - - except (httpx.TimeoutException, httpx.RequestError) as e: - retry_count += 1 - if retry_count > self.max_retries: - final_status = "timeout" - final_retry_count = retry_count - raise httpx.TimeoutException( - f"Failed to load data after {self.max_retries} retries: {str(e)}", - request=e.request - if isinstance(e, httpx.RequestError) - else None, + return b"" + response.raise_for_status() + content = response.content + response_bytes = len(content) + final_retry_count = retry_count + if headers: + if response.status_code == httpx.codes.OK: + logger.debug( + "Gateway ignored Range request for CID %s; " + "slicing the complete response locally", + cid, + ) + return _slice_requested_range( + content, offset, length, suffix + ) + if response.status_code == httpx.codes.PARTIAL_CONTENT: + # Trust the partial body only after proving its + # Content-Range matches the requested byte window. + _validate_partial_content( + response, offset, length, suffix, response_bytes ) + return content + # Any other 2xx to a Range request is unexpected: we + # cannot know which bytes it carries, so fail rather than + # return a possibly-wrong window. + raise httpx.HTTPStatusError( + f"unexpected {response.status_code} response to a " + "Range request", + request=response.request, + response=response, + ) + return content - # Calculate backoff delay with jitter - delay = self.initial_delay * ( - self.backoff_factor ** (retry_count - 1) + except httpx.RequestError: + if retry_count >= self.max_retries: + final_status = "request_error" + final_retry_count = retry_count + raise + retry_count += 1 + await asyncio.sleep( + _retry_delay( + self.initial_delay, self.backoff_factor, retry_count ) - jitter = delay * 0.1 * (random.random() - 0.5) - await asyncio.sleep(delay + jitter) + ) - except httpx.HTTPStatusError: - # Re-raise non-timeout HTTP errors immediately + except httpx.HTTPStatusError as error: + if ( + error.response.status_code not in _RETRYABLE_STATUS_CODES + or retry_count >= self.max_retries + ): final_status = "http_error" final_retry_count = retry_count raise + retry_count += 1 + await asyncio.sleep( + _retry_delay( + self.initial_delay, + self.backoff_factor, + retry_count, + error.response, + ) + ) finally: instrumentation.end_cas_load( trace_started_at, @@ -525,7 +916,7 @@ async def load( raise RuntimeError("Exited the retry loop unexpectedly.") # pragma: no cover # --------------------------------------------------------------------- # - # pin_cid() – method to pin a CID # + # pin_cid() - method to pin a CID # # --------------------------------------------------------------------- # async def pin_cid( self, @@ -544,7 +935,7 @@ async def pin_cid( params = {"arg": str(cid), "recursive": "true"} pin_add_url_base: str = f"{target_rpc}/api/v0/pin/add" - async with self._sem: # throttle RPC + async with self._loop_semaphore(): # throttle RPC client = self._loop_client() response = await client.post(pin_add_url_base, params=params) response.raise_for_status() @@ -560,7 +951,7 @@ async def unpin_cid( """ params = {"arg": str(cid), "recursive": "true"} unpin_url_base: str = f"{target_rpc}/api/v0/pin/rm" - async with self._sem: # throttle RPC + async with self._loop_semaphore(): # throttle RPC client = self._loop_client() response = await client.post(unpin_url_base, params=params) response.raise_for_status() @@ -580,7 +971,7 @@ async def pin_update( """ params = {"arg": [str(old_id), str(new_id)]} pin_update_url_base: str = f"{target_rpc}/api/v0/pin/update" - async with self._sem: # throttle RPC + async with self._loop_semaphore(): # throttle RPC client = self._loop_client() response = await client.post(pin_update_url_base, params=params) response.raise_for_status() @@ -598,7 +989,7 @@ async def pin_ls( List[CID]: A list of pinned CIDs. """ pin_ls_url_base: str = f"{target_rpc}/api/v0/pin/ls" - async with self._sem: # throttle RPC + async with self._loop_semaphore(): # throttle RPC client = self._loop_client() response = await client.post(pin_ls_url_base) response.raise_for_status() diff --git a/py_hamt/zarr_hamt_store.py b/py_hamt/zarr_hamt_store.py index 2984e78..4c1a8c6 100644 --- a/py_hamt/zarr_hamt_store.py +++ b/py_hamt/zarr_hamt_store.py @@ -234,13 +234,14 @@ def supports_partial_writes(self) -> bool: return False async def set(self, key: str, value: zarr.core.buffer.Buffer) -> None: - """@private""" + """Store a value and update any existing metadata cache entry.""" if self.read_only: raise Exception("Cannot write to a read-only store.") + raw_bytes = value.to_bytes() + await self.hamt.set(key, raw_bytes) if key in self.metadata_read_cache: - self.metadata_read_cache[key] = value.to_bytes() - await self.hamt.set(key, value.to_bytes()) + self.metadata_read_cache[key] = raw_bytes async def set_if_not_exists(self, key: str, value: zarr.core.buffer.Buffer) -> None: """@private""" @@ -259,18 +260,16 @@ def supports_deletes(self) -> bool: return not self.hamt.read_only async def delete(self, key: str) -> None: - """@private""" + """Delete a key and evict any cached metadata for it.""" if self.read_only: raise Exception("Cannot write to a read-only store.") try: await self.hamt.delete(key) - # In practice these lines never seem to be needed, creating and appending data are the only operations most zarrs actually undergo - # if key in self.metadata_read_cache: - # del self.metadata_read_cache[key] # It's fine if the key was not in the HAMT # Sometimes zarr v3 calls deletes on keys that don't exist (or have already been deleted) for some reason, probably concurrency issues except KeyError: - return + pass + self.metadata_read_cache.pop(key, None) @property def supports_listing(self) -> bool: diff --git a/pyproject.toml b/pyproject.toml index 5c946e0..a49d045 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "py-hamt" -version = "3.4.1" +version = "3.5.0" description = "HAMT implementation for a content-addressed storage system." readme = "README.md" requires-python = ">=3.12" @@ -36,6 +36,7 @@ dev = [ "docker>=7.1.0", "types-docker>=7.1.0.20250523", "pre-commit>=4.2.0", + "trustme>=1.2.1", ] [tool.ruff] diff --git a/pytest.ini b/pytest.ini index e21418c..752e30f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ # pytest.ini [pytest] asyncio_mode = auto -asyncio_default_fixture_loop_scope = session +asyncio_default_fixture_loop_scope = function filterwarnings = ignore:sharded_zarr_v1 is deprecated.*:FutureWarning diff --git a/tests/conftest.py b/tests/conftest.py index f38e01e..67a63fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,9 +5,11 @@ from testing_utils import create_ipfs, ipld_strategy # noqa: F401 -@pytest.fixture(scope="session") +@pytest.fixture async def global_client_session(): - """One httpx.AsyncClient shared by the whole test run.""" + """A fresh httpx.AsyncClient per test (function-scoped to match the + function-scoped event loop; a session-scoped client bound to one test's + loop breaks later tests once that loop closes).""" async with httpx.AsyncClient() as client: yield client # httpx's async context manager awaits client.aclose() for us diff --git a/tests/fixtures/z1_legacy_multivar_store.json b/tests/fixtures/z1_legacy_multivar_store.json new file mode 100644 index 0000000..40c8ce5 --- /dev/null +++ b/tests/fixtures/z1_legacy_multivar_store.json @@ -0,0 +1,21 @@ +{ + "blocks": { + "bafkr4ia4me6go7iwqqwh6ricscq6w7qy6vtjmwcmc6d5loeyrnttlycdtq": "KLUv/WCAAH0AACAAAEAAAwBuKwoAxC3AAg==", + "bafkr4iamm5aavgo5molhsg5z4f254ajp3kz3khdmvrzueombleskssbg7a": "ewogICJhdHRyaWJ1dGVzIjoge30sCiAgInphcnJfZm9ybWF0IjogMywKICAiY29uc29saWRhdGVkX21ldGFkYXRhIjogbnVsbCwKICAibm9kZV90eXBlIjogImdyb3VwIgp9", + "bafkr4ibtjlcorwfnwkqrxhjay3fhbmeujunlp5xm4tpkquysc6oak3xumq": "ewogICJhdHRyaWJ1dGVzIjoge30sCiAgInphcnJfZm9ybWF0IjogMywKICAiY29uc29saWRhdGVkX21ldGFkYXRhIjogewogICAgImtpbmQiOiAiaW5saW5lIiwKICAgICJtdXN0X3VuZGVyc3RhbmQiOiBmYWxzZSwKICAgICJtZXRhZGF0YSI6IHsKICAgICAgInRlbXAiOiB7CiAgICAgICAgInNoYXBlIjogWwogICAgICAgICAgNCwKICAgICAgICAgIDQsCiAgICAgICAgICA2CiAgICAgICAgXSwKICAgICAgICAiZGF0YV90eXBlIjogImZsb2F0NjQiLAogICAgICAgICJjaHVua19ncmlkIjogewogICAgICAgICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgICAgICAgIDIsCiAgICAgICAgICAgICAgNCwKICAgICAgICAgICAgICA2CiAgICAgICAgICAgIF0KICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJjaHVua19rZXlfZW5jb2RpbmciOiB7CiAgICAgICAgICAibmFtZSI6ICJkZWZhdWx0IiwKICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAic2VwYXJhdG9yIjogIi8iCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICAiZmlsbF92YWx1ZSI6IDAuMCwKICAgICAgICAiY29kZWNzIjogWwogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICAgICAgICB9CiAgICAgICAgICB9LAogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImxldmVsIjogMCwKICAgICAgICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICAgICAgICB9CiAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICAiYXR0cmlidXRlcyI6IHsKICAgICAgICAgICJfRmlsbFZhbHVlIjogIkFBQUFBQUFBK0g4PSIKICAgICAgICB9LAogICAgICAgICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAgICAgICAidGltZSIsCiAgICAgICAgICAibGF0IiwKICAgICAgICAgICJsb24iCiAgICAgICAgXSwKICAgICAgICAiemFycl9mb3JtYXQiOiAzLAogICAgICAgICJub2RlX3R5cGUiOiAiYXJyYXkiLAogICAgICAgICJzdG9yYWdlX3RyYW5zZm9ybWVycyI6IFtdCiAgICAgIH0sCiAgICAgICJ0aW1lIjogewogICAgICAgICJzaGFwZSI6IFsKICAgICAgICAgIDQKICAgICAgICBdLAogICAgICAgICJkYXRhX3R5cGUiOiAiaW50NjQiLAogICAgICAgICJjaHVua19ncmlkIjogewogICAgICAgICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgICAgICAgIDQKICAgICAgICAgICAgXQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgImNodW5rX2tleV9lbmNvZGluZyI6IHsKICAgICAgICAgICJuYW1lIjogImRlZmF1bHQiLAogICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICJzZXBhcmF0b3IiOiAiLyIKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJmaWxsX3ZhbHVlIjogMCwKICAgICAgICAiY29kZWNzIjogWwogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICAgICAgICB9CiAgICAgICAgICB9LAogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImxldmVsIjogMCwKICAgICAgICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICAgICAgICB9CiAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICAiYXR0cmlidXRlcyI6IHt9LAogICAgICAgICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAgICAgICAidGltZSIKICAgICAgICBdLAogICAgICAgICJ6YXJyX2Zvcm1hdCI6IDMsCiAgICAgICAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgICAgICAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KICAgICAgfSwKICAgICAgInByZWNpcCI6IHsKICAgICAgICAic2hhcGUiOiBbCiAgICAgICAgICA0LAogICAgICAgICAgNCwKICAgICAgICAgIDYKICAgICAgICBdLAogICAgICAgICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgICAgICAgImNodW5rX2dyaWQiOiB7CiAgICAgICAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgICAgICAgMiwKICAgICAgICAgICAgICA0LAogICAgICAgICAgICAgIDYKICAgICAgICAgICAgXQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgImNodW5rX2tleV9lbmNvZGluZyI6IHsKICAgICAgICAgICJuYW1lIjogImRlZmF1bHQiLAogICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICJzZXBhcmF0b3IiOiAiLyIKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJmaWxsX3ZhbHVlIjogMC4wLAogICAgICAgICJjb2RlY3MiOiBbCiAgICAgICAgICB7CiAgICAgICAgICAgICJuYW1lIjogImJ5dGVzIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImVuZGlhbiI6ICJsaXR0bGUiCiAgICAgICAgICAgIH0KICAgICAgICAgIH0sCiAgICAgICAgICB7CiAgICAgICAgICAgICJuYW1lIjogInpzdGQiLAogICAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgICAibGV2ZWwiOiAwLAogICAgICAgICAgICAgICJjaGVja3N1bSI6IGZhbHNlCiAgICAgICAgICAgIH0KICAgICAgICAgIH0KICAgICAgICBdLAogICAgICAgICJhdHRyaWJ1dGVzIjogewogICAgICAgICAgIl9GaWxsVmFsdWUiOiAiQUFBQUFBQUErSDg9IgogICAgICAgIH0sCiAgICAgICAgImRpbWVuc2lvbl9uYW1lcyI6IFsKICAgICAgICAgICJ0aW1lIiwKICAgICAgICAgICJsYXQiLAogICAgICAgICAgImxvbiIKICAgICAgICBdLAogICAgICAgICJ6YXJyX2Zvcm1hdCI6IDMsCiAgICAgICAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgICAgICAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KICAgICAgfSwKICAgICAgImxhdCI6IHsKICAgICAgICAic2hhcGUiOiBbCiAgICAgICAgICA0CiAgICAgICAgXSwKICAgICAgICAiZGF0YV90eXBlIjogImZsb2F0NjQiLAogICAgICAgICJjaHVua19ncmlkIjogewogICAgICAgICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgICAgICAgIDQKICAgICAgICAgICAgXQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgImNodW5rX2tleV9lbmNvZGluZyI6IHsKICAgICAgICAgICJuYW1lIjogImRlZmF1bHQiLAogICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICJzZXBhcmF0b3IiOiAiLyIKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJmaWxsX3ZhbHVlIjogMC4wLAogICAgICAgICJjb2RlY3MiOiBbCiAgICAgICAgICB7CiAgICAgICAgICAgICJuYW1lIjogImJ5dGVzIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImVuZGlhbiI6ICJsaXR0bGUiCiAgICAgICAgICAgIH0KICAgICAgICAgIH0sCiAgICAgICAgICB7CiAgICAgICAgICAgICJuYW1lIjogInpzdGQiLAogICAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgICAibGV2ZWwiOiAwLAogICAgICAgICAgICAgICJjaGVja3N1bSI6IGZhbHNlCiAgICAgICAgICAgIH0KICAgICAgICAgIH0KICAgICAgICBdLAogICAgICAgICJhdHRyaWJ1dGVzIjogewogICAgICAgICAgIl9GaWxsVmFsdWUiOiAiQUFBQUFBQUErSDg9IgogICAgICAgIH0sCiAgICAgICAgImRpbWVuc2lvbl9uYW1lcyI6IFsKICAgICAgICAgICJsYXQiCiAgICAgICAgXSwKICAgICAgICAiemFycl9mb3JtYXQiOiAzLAogICAgICAgICJub2RlX3R5cGUiOiAiYXJyYXkiLAogICAgICAgICJzdG9yYWdlX3RyYW5zZm9ybWVycyI6IFtdCiAgICAgIH0sCiAgICAgICJsb24iOiB7CiAgICAgICAgInNoYXBlIjogWwogICAgICAgICAgNgogICAgICAgIF0sCiAgICAgICAgImRhdGFfdHlwZSI6ICJmbG9hdDY0IiwKICAgICAgICAiY2h1bmtfZ3JpZCI6IHsKICAgICAgICAgICJuYW1lIjogInJlZ3VsYXIiLAogICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICJjaHVua19zaGFwZSI6IFsKICAgICAgICAgICAgICA2CiAgICAgICAgICAgIF0KICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJjaHVua19rZXlfZW5jb2RpbmciOiB7CiAgICAgICAgICAibmFtZSI6ICJkZWZhdWx0IiwKICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAic2VwYXJhdG9yIjogIi8iCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICAiZmlsbF92YWx1ZSI6IDAuMCwKICAgICAgICAiY29kZWNzIjogWwogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICAgICAgICB9CiAgICAgICAgICB9LAogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImxldmVsIjogMCwKICAgICAgICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICAgICAgICB9CiAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICAiYXR0cmlidXRlcyI6IHsKICAgICAgICAgICJfRmlsbFZhbHVlIjogIkFBQUFBQUFBK0g4PSIKICAgICAgICB9LAogICAgICAgICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAgICAgICAibG9uIgogICAgICAgIF0sCiAgICAgICAgInphcnJfZm9ybWF0IjogMywKICAgICAgICAibm9kZV90eXBlIjogImFycmF5IiwKICAgICAgICAic3RvcmFnZV90cmFuc2Zvcm1lcnMiOiBbXQogICAgICB9CiAgICB9CiAgfSwKICAibm9kZV90eXBlIjogImdyb3VwIgp9", + "bafkr4ick6fvm7ckmt2c6rfyl3aajcx3p3mdvugrihlinfmui3c5h2uei5q": "ewogICJzaGFwZSI6IFsKICAgIDQsCiAgICA0LAogICAgNgogIF0sCiAgImRhdGFfdHlwZSI6ICJmbG9hdDY0IiwKICAiY2h1bmtfZ3JpZCI6IHsKICAgICJuYW1lIjogInJlZ3VsYXIiLAogICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICJjaHVua19zaGFwZSI6IFsKICAgICAgICAyLAogICAgICAgIDQsCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7fSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgInRpbWUiLAogICAgImxhdCIsCiAgICAibG9uIgogIF0sCiAgInphcnJfZm9ybWF0IjogMywKICAibm9kZV90eXBlIjogImFycmF5IiwKICAic3RvcmFnZV90cmFuc2Zvcm1lcnMiOiBbXQp9", + "bafkr4icpnkpo3pserbi4nekr2ldezdajzdc2z2kq2auxkqvs5pv5lnnvxa": "ewogICJzaGFwZSI6IFsKICAgIDQsCiAgICA0LAogICAgNgogIF0sCiAgImRhdGFfdHlwZSI6ICJmbG9hdDY0IiwKICAiY2h1bmtfZ3JpZCI6IHsKICAgICJuYW1lIjogInJlZ3VsYXIiLAogICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICJjaHVua19zaGFwZSI6IFsKICAgICAgICAyLAogICAgICAgIDQsCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7CiAgICAiX0ZpbGxWYWx1ZSI6ICJBQUFBQUFBQStIOD0iCiAgfSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgInRpbWUiLAogICAgImxhdCIsCiAgICAibG9uIgogIF0sCiAgInphcnJfZm9ybWF0IjogMywKICAibm9kZV90eXBlIjogImFycmF5IiwKICAic3RvcmFnZV90cmFuc2Zvcm1lcnMiOiBbXQp9", + "bafkr4iczmsrm27icvtifxcvwogorby6a32jhv4r4sall3cvqvfvpud4kmu": "ewogICJzaGFwZSI6IFsKICAgIDQKICBdLAogICJkYXRhX3R5cGUiOiAiaW50NjQiLAogICJjaHVua19ncmlkIjogewogICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgIDQKICAgICAgXQogICAgfQogIH0sCiAgImNodW5rX2tleV9lbmNvZGluZyI6IHsKICAgICJuYW1lIjogImRlZmF1bHQiLAogICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICJzZXBhcmF0b3IiOiAiLyIKICAgIH0KICB9LAogICJmaWxsX3ZhbHVlIjogMCwKICAiY29kZWNzIjogWwogICAgewogICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICB9CiAgICB9LAogICAgewogICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgImxldmVsIjogMCwKICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICB9CiAgICB9CiAgXSwKICAiYXR0cmlidXRlcyI6IHt9LAogICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAidGltZSIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4idbvkz7l3nb376a6hkrn4jhfeiakhx7bohuqhtin3selkns2vjnt4": "ewogICJzaGFwZSI6IFsKICAgIDYKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7CiAgICAiX0ZpbGxWYWx1ZSI6ICJBQUFBQUFBQStIOD0iCiAgfSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxvbiIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4idj3cpeoiwwyy5rdpx7tidnlehl3etnb77dbxvkferhp2cwyiwpve": "ewogICJzaGFwZSI6IFsKICAgIDQKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNAogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7CiAgICAiX0ZpbGxWYWx1ZSI6ICJBQUFBQUFBQStIOD0iCiAgfSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxhdCIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4iejghfjnobiycewyqcqbr5zcjskim4lc5t3czkqjo2vcmopq5y2sa": "KLUv/SAgrQAAcAAAAQACAAMAAAAAAAAAA1QCAAMB", + "bafkr4iekekhcnomg3irwttcrotdo2ztgzjffxiaixwvan4tgindj7xebdy": "KLUv/SAw7QAAgAAAAAAAgGbAAFtCQFuAZkAFAGAIGAcIW0nRgiM=", + "bafkr4ifysp246qb32nooik7bsg3xqc5olxqxdhy2dibb4c3xxxpakqy36e": "ewogICJzaGFwZSI6IFsKICAgIDQKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNAogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7fSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxhdCIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4ihm54bteuyux6jefpw4tvudkavs6fshntrh7yalpslzs4l525qeuu": "KLUv/WCAAIUAACgAAPA/AAMAbysKABkBwAI=", + "bafkr4ihmv4rapvqddmvhiy3l2zc5vnlavxnjwiqhpgr26q7723ymyi6c74": "ewogICJzaGFwZSI6IFsKICAgIDYKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7fSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxvbiIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4ihwfbcpgt5veqn7hw2pgzkf6z34nrhc4hkscreq7z4bqn2zhqg65e": "KLUv/SAgxQAAcAAAAAAAgFbAAD5AgFZAAwBAyArdCxwB", + "bafyr4ibicxbn7hn5ppkbt7bcapsi55bgm5iq4ah2aibehigpxauacyfot4": "iNgqWCUAAVUeIOzvAzJTFL+SQr7cnWg1ArLxZHbOJ/4At8l5lxfddgSl2CpYJQABVR4g7O8DMlMUv5JCvtydaDUCsvFkds4n/gC3yXmXF912BKX29vb29vY=", + "bafyr4ifhwc3ydk6yswt4ljyz57tkesvkizmpkc4caf3af2l6p6nyiz7jcu": "o2ZjaHVua3OlanNoYXJkX2NpZHOB2CpYJQABcR4gKBXC3529e9QZ/CID5I70JmdRDgD6AgJDoM+4KAFgrp9rYXJyYXlfc2hhcGWDBAQGa2NodW5rX3NoYXBlgwIEBm9zaGFyZGluZ19jb25maWehcGNodW5rc19wZXJfc2hhcmQIcnByaW1hcnlfYXJyYXlfcGF0aGR0ZW1waG1ldGFkYXRhq2dsYXQvYy8w2CpYJQABVR4g9ihE80+1JBvz2082VF9nfGxOLh1SFEkP54GDdZPA3ulnbG9uL2MvMNgqWCUAAVUeIIoijia5htojacxRdMbtZmbKSlugCL2qBvJmQ0af3IEeaHRpbWUvYy8w2CpYJQABVR4giTHKlrgowIlsQFAMe5EmSkM4sXZ7FlUEu1UTHPh3GpBpemFyci5qc29u2CpYJQABVR4gM0rE6NitsqEbnSDGynCwlE0at/bs5N6oUxIXnAVu9GRtbGF0L3phcnIuanNvbtgqWCUAAVUeIGnYnkci1sY7Eb7/mgbVkOvZJtD/4w3qopInfoVsIs+pbWxvbi96YXJyLmpzb27YKlglAAFVHiBhqrP17aHf/A8dUW8ScpEAUe/wuPSB5obuRFqbLVUtn25wcmVjaXAvYy8wLzAvMNgqWCUAAVUeIBxhPGd9FoQsf0UCkKHrfhj1ZpZYTBeH1biYi2c14EOcbnByZWNpcC9jLzEvMC8w2CpYJQABVR4gHGE8Z30WhCx/RQKQoet+GPVmllhMF4fVuJiLZzXgQ5xudGVtcC96YXJyLmpzb27YKlglAAFVHiBPap7tvkSIUcaRUdLGTIwJyMWs6VDQKXVCsuvr1bW1uG50aW1lL3phcnIuanNvbtgqWCUAAVUeIFlkos19AqzQW4q2cZ0Q48DeknryPJAWvYqwqWr6D4plcHByZWNpcC96YXJyLmpzb27YKlglAAFVHiBPap7tvkSIUcaRUdLGTIwJyMWs6VDQKXVCsuvr1bW1uHBtYW5pZmVzdF92ZXJzaW9ub3NoYXJkZWRfemFycl92MQ==" + }, + "root_cid": "bafyr4ifhwc3ydk6yswt4ljyz57tkesvkizmpkc4caf3af2l6p6nyiz7jcu" +} diff --git a/tests/fixtures/z1_legacy_single_var_store.json b/tests/fixtures/z1_legacy_single_var_store.json new file mode 100644 index 0000000..7c1fd41 --- /dev/null +++ b/tests/fixtures/z1_legacy_single_var_store.json @@ -0,0 +1,21 @@ +{ + "blocks": { + "bafkr4iamm5aavgo5molhsg5z4f254ajp3kz3khdmvrzueombleskssbg7a": "ewogICJhdHRyaWJ1dGVzIjoge30sCiAgInphcnJfZm9ybWF0IjogMywKICAiY29uc29saWRhdGVkX21ldGFkYXRhIjogbnVsbCwKICAibm9kZV90eXBlIjogImdyb3VwIgp9", + "bafkr4ibxty4gbihdefvbos7sjo75crb7el3rmbd7rsjvh5n5p3agetbona": "KLUv/WCAAO0DANQDAADwPwBAAAgQFBgcICIkJigqLC4wMTIzNDU2Nzg5Ojs8PT4/QIAAQYAAQoAAQ4AARIAARYAARoAAR4BHQC8g4NcDMxsti4GZjZbFwMxGy2JgZkPyZZYln8yy5MssS77MsuTLLEu+zLLkyyxLvsyyyJdZFpFvyVrDHPfa", + "bafkr4ick6fvm7ckmt2c6rfyl3aajcx3p3mdvugrihlinfmui3c5h2uei5q": "ewogICJzaGFwZSI6IFsKICAgIDQsCiAgICA0LAogICAgNgogIF0sCiAgImRhdGFfdHlwZSI6ICJmbG9hdDY0IiwKICAiY2h1bmtfZ3JpZCI6IHsKICAgICJuYW1lIjogInJlZ3VsYXIiLAogICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICJjaHVua19zaGFwZSI6IFsKICAgICAgICAyLAogICAgICAgIDQsCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7fSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgInRpbWUiLAogICAgImxhdCIsCiAgICAibG9uIgogIF0sCiAgInphcnJfZm9ybWF0IjogMywKICAibm9kZV90eXBlIjogImFycmF5IiwKICAic3RvcmFnZV90cmFuc2Zvcm1lcnMiOiBbXQp9", + "bafkr4icpnkpo3pserbi4nekr2ldezdajzdc2z2kq2auxkqvs5pv5lnnvxa": "ewogICJzaGFwZSI6IFsKICAgIDQsCiAgICA0LAogICAgNgogIF0sCiAgImRhdGFfdHlwZSI6ICJmbG9hdDY0IiwKICAiY2h1bmtfZ3JpZCI6IHsKICAgICJuYW1lIjogInJlZ3VsYXIiLAogICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICJjaHVua19zaGFwZSI6IFsKICAgICAgICAyLAogICAgICAgIDQsCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7CiAgICAiX0ZpbGxWYWx1ZSI6ICJBQUFBQUFBQStIOD0iCiAgfSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgInRpbWUiLAogICAgImxhdCIsCiAgICAibG9uIgogIF0sCiAgInphcnJfZm9ybWF0IjogMywKICAibm9kZV90eXBlIjogImFycmF5IiwKICAic3RvcmFnZV90cmFuc2Zvcm1lcnMiOiBbXQp9", + "bafkr4iczmsrm27icvtifxcvwogorby6a32jhv4r4sall3cvqvfvpud4kmu": "ewogICJzaGFwZSI6IFsKICAgIDQKICBdLAogICJkYXRhX3R5cGUiOiAiaW50NjQiLAogICJjaHVua19ncmlkIjogewogICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgIDQKICAgICAgXQogICAgfQogIH0sCiAgImNodW5rX2tleV9lbmNvZGluZyI6IHsKICAgICJuYW1lIjogImRlZmF1bHQiLAogICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICJzZXBhcmF0b3IiOiAiLyIKICAgIH0KICB9LAogICJmaWxsX3ZhbHVlIjogMCwKICAiY29kZWNzIjogWwogICAgewogICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICB9CiAgICB9LAogICAgewogICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgImxldmVsIjogMCwKICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICB9CiAgICB9CiAgXSwKICAiYXR0cmlidXRlcyI6IHt9LAogICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAidGltZSIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4idbvkz7l3nb376a6hkrn4jhfeiakhx7bohuqhtin3selkns2vjnt4": "ewogICJzaGFwZSI6IFsKICAgIDYKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7CiAgICAiX0ZpbGxWYWx1ZSI6ICJBQUFBQUFBQStIOD0iCiAgfSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxvbiIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4idj3cpeoiwwyy5rdpx7tidnlehl3etnb77dbxvkferhp2cwyiwpve": "ewogICJzaGFwZSI6IFsKICAgIDQKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNAogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7CiAgICAiX0ZpbGxWYWx1ZSI6ICJBQUFBQUFBQStIOD0iCiAgfSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxhdCIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4iejghfjnobiycewyqcqbr5zcjskim4lc5t3czkqjo2vcmopq5y2sa": "KLUv/SAgrQAAcAAAAQACAAMAAAAAAAAAA1QCAAMB", + "bafkr4iekekhcnomg3irwttcrotdo2ztgzjffxiaixwvan4tgindj7xebdy": "KLUv/SAw7QAAgAAAAAAAgGbAAFtCQFuAZkAFAGAIGAcIW0nRgiM=", + "bafkr4iem6twzkgh63qzeph75im4udb3xdbkfnnda62j6r2myc4p6tpwpzq": "KLUv/WCAAL0DAFIEDRawNcMBDMMAkzlnm3UTERER0UFjGM5TOlvHrYBbALcatxi3FrcUtxJH/HDDCyd8cMFVoAEwIODXWfIdmFmWfAdmliXfgZllyXdgZlnyHSizLPkOzCxLvgMzy5LvwMxGy2JgZqNlMVBmo2UxMDMy2lLYsBoW", + "bafkr4ifhaszbondcaw3nvy7lbco56htp7ogybo45iqdl6i4bfqedtlpine": "ewogICJhdHRyaWJ1dGVzIjoge30sCiAgInphcnJfZm9ybWF0IjogMywKICAiY29uc29saWRhdGVkX21ldGFkYXRhIjogewogICAgImtpbmQiOiAiaW5saW5lIiwKICAgICJtdXN0X3VuZGVyc3RhbmQiOiBmYWxzZSwKICAgICJtZXRhZGF0YSI6IHsKICAgICAgInRlbXAiOiB7CiAgICAgICAgInNoYXBlIjogWwogICAgICAgICAgNCwKICAgICAgICAgIDQsCiAgICAgICAgICA2CiAgICAgICAgXSwKICAgICAgICAiZGF0YV90eXBlIjogImZsb2F0NjQiLAogICAgICAgICJjaHVua19ncmlkIjogewogICAgICAgICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgICAgICAgIDIsCiAgICAgICAgICAgICAgNCwKICAgICAgICAgICAgICA2CiAgICAgICAgICAgIF0KICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJjaHVua19rZXlfZW5jb2RpbmciOiB7CiAgICAgICAgICAibmFtZSI6ICJkZWZhdWx0IiwKICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAic2VwYXJhdG9yIjogIi8iCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICAiZmlsbF92YWx1ZSI6IDAuMCwKICAgICAgICAiY29kZWNzIjogWwogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICAgICAgICB9CiAgICAgICAgICB9LAogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImxldmVsIjogMCwKICAgICAgICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICAgICAgICB9CiAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICAiYXR0cmlidXRlcyI6IHsKICAgICAgICAgICJfRmlsbFZhbHVlIjogIkFBQUFBQUFBK0g4PSIKICAgICAgICB9LAogICAgICAgICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAgICAgICAidGltZSIsCiAgICAgICAgICAibGF0IiwKICAgICAgICAgICJsb24iCiAgICAgICAgXSwKICAgICAgICAiemFycl9mb3JtYXQiOiAzLAogICAgICAgICJub2RlX3R5cGUiOiAiYXJyYXkiLAogICAgICAgICJzdG9yYWdlX3RyYW5zZm9ybWVycyI6IFtdCiAgICAgIH0sCiAgICAgICJ0aW1lIjogewogICAgICAgICJzaGFwZSI6IFsKICAgICAgICAgIDQKICAgICAgICBdLAogICAgICAgICJkYXRhX3R5cGUiOiAiaW50NjQiLAogICAgICAgICJjaHVua19ncmlkIjogewogICAgICAgICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgICAgICAgIDQKICAgICAgICAgICAgXQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgImNodW5rX2tleV9lbmNvZGluZyI6IHsKICAgICAgICAgICJuYW1lIjogImRlZmF1bHQiLAogICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICJzZXBhcmF0b3IiOiAiLyIKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJmaWxsX3ZhbHVlIjogMCwKICAgICAgICAiY29kZWNzIjogWwogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICAgICAgICB9CiAgICAgICAgICB9LAogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImxldmVsIjogMCwKICAgICAgICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICAgICAgICB9CiAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICAiYXR0cmlidXRlcyI6IHt9LAogICAgICAgICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAgICAgICAidGltZSIKICAgICAgICBdLAogICAgICAgICJ6YXJyX2Zvcm1hdCI6IDMsCiAgICAgICAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgICAgICAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KICAgICAgfSwKICAgICAgImxhdCI6IHsKICAgICAgICAic2hhcGUiOiBbCiAgICAgICAgICA0CiAgICAgICAgXSwKICAgICAgICAiZGF0YV90eXBlIjogImZsb2F0NjQiLAogICAgICAgICJjaHVua19ncmlkIjogewogICAgICAgICAgIm5hbWUiOiAicmVndWxhciIsCiAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImNodW5rX3NoYXBlIjogWwogICAgICAgICAgICAgIDQKICAgICAgICAgICAgXQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgImNodW5rX2tleV9lbmNvZGluZyI6IHsKICAgICAgICAgICJuYW1lIjogImRlZmF1bHQiLAogICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICJzZXBhcmF0b3IiOiAiLyIKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJmaWxsX3ZhbHVlIjogMC4wLAogICAgICAgICJjb2RlY3MiOiBbCiAgICAgICAgICB7CiAgICAgICAgICAgICJuYW1lIjogImJ5dGVzIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImVuZGlhbiI6ICJsaXR0bGUiCiAgICAgICAgICAgIH0KICAgICAgICAgIH0sCiAgICAgICAgICB7CiAgICAgICAgICAgICJuYW1lIjogInpzdGQiLAogICAgICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgICAibGV2ZWwiOiAwLAogICAgICAgICAgICAgICJjaGVja3N1bSI6IGZhbHNlCiAgICAgICAgICAgIH0KICAgICAgICAgIH0KICAgICAgICBdLAogICAgICAgICJhdHRyaWJ1dGVzIjogewogICAgICAgICAgIl9GaWxsVmFsdWUiOiAiQUFBQUFBQUErSDg9IgogICAgICAgIH0sCiAgICAgICAgImRpbWVuc2lvbl9uYW1lcyI6IFsKICAgICAgICAgICJsYXQiCiAgICAgICAgXSwKICAgICAgICAiemFycl9mb3JtYXQiOiAzLAogICAgICAgICJub2RlX3R5cGUiOiAiYXJyYXkiLAogICAgICAgICJzdG9yYWdlX3RyYW5zZm9ybWVycyI6IFtdCiAgICAgIH0sCiAgICAgICJsb24iOiB7CiAgICAgICAgInNoYXBlIjogWwogICAgICAgICAgNgogICAgICAgIF0sCiAgICAgICAgImRhdGFfdHlwZSI6ICJmbG9hdDY0IiwKICAgICAgICAiY2h1bmtfZ3JpZCI6IHsKICAgICAgICAgICJuYW1lIjogInJlZ3VsYXIiLAogICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICJjaHVua19zaGFwZSI6IFsKICAgICAgICAgICAgICA2CiAgICAgICAgICAgIF0KICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJjaHVua19rZXlfZW5jb2RpbmciOiB7CiAgICAgICAgICAibmFtZSI6ICJkZWZhdWx0IiwKICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAic2VwYXJhdG9yIjogIi8iCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICAiZmlsbF92YWx1ZSI6IDAuMCwKICAgICAgICAiY29kZWNzIjogWwogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJieXRlcyIsCiAgICAgICAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICAgICAgICJlbmRpYW4iOiAibGl0dGxlIgogICAgICAgICAgICB9CiAgICAgICAgICB9LAogICAgICAgICAgewogICAgICAgICAgICAibmFtZSI6ICJ6c3RkIiwKICAgICAgICAgICAgImNvbmZpZ3VyYXRpb24iOiB7CiAgICAgICAgICAgICAgImxldmVsIjogMCwKICAgICAgICAgICAgICAiY2hlY2tzdW0iOiBmYWxzZQogICAgICAgICAgICB9CiAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICAiYXR0cmlidXRlcyI6IHsKICAgICAgICAgICJfRmlsbFZhbHVlIjogIkFBQUFBQUFBK0g4PSIKICAgICAgICB9LAogICAgICAgICJkaW1lbnNpb25fbmFtZXMiOiBbCiAgICAgICAgICAibG9uIgogICAgICAgIF0sCiAgICAgICAgInphcnJfZm9ybWF0IjogMywKICAgICAgICAibm9kZV90eXBlIjogImFycmF5IiwKICAgICAgICAic3RvcmFnZV90cmFuc2Zvcm1lcnMiOiBbXQogICAgICB9CiAgICB9CiAgfSwKICAibm9kZV90eXBlIjogImdyb3VwIgp9", + "bafkr4ifysp246qb32nooik7bsg3xqc5olxqxdhy2dibb4c3xxxpakqy36e": "ewogICJzaGFwZSI6IFsKICAgIDQKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNAogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7fSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxhdCIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4ihmv4rapvqddmvhiy3l2zc5vnlavxnjwiqhpgr26q7723ymyi6c74": "ewogICJzaGFwZSI6IFsKICAgIDYKICBdLAogICJkYXRhX3R5cGUiOiAiZmxvYXQ2NCIsCiAgImNodW5rX2dyaWQiOiB7CiAgICAibmFtZSI6ICJyZWd1bGFyIiwKICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAiY2h1bmtfc2hhcGUiOiBbCiAgICAgICAgNgogICAgICBdCiAgICB9CiAgfSwKICAiY2h1bmtfa2V5X2VuY29kaW5nIjogewogICAgIm5hbWUiOiAiZGVmYXVsdCIsCiAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgInNlcGFyYXRvciI6ICIvIgogICAgfQogIH0sCiAgImZpbGxfdmFsdWUiOiAwLjAsCiAgImNvZGVjcyI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYnl0ZXMiLAogICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAiZW5kaWFuIjogImxpdHRsZSIKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm5hbWUiOiAienN0ZCIsCiAgICAgICJjb25maWd1cmF0aW9uIjogewogICAgICAgICJsZXZlbCI6IDAsCiAgICAgICAgImNoZWNrc3VtIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImF0dHJpYnV0ZXMiOiB7fSwKICAiZGltZW5zaW9uX25hbWVzIjogWwogICAgImxvbiIKICBdLAogICJ6YXJyX2Zvcm1hdCI6IDMsCiAgIm5vZGVfdHlwZSI6ICJhcnJheSIsCiAgInN0b3JhZ2VfdHJhbnNmb3JtZXJzIjogW10KfQ==", + "bafkr4ihwfbcpgt5veqn7hw2pgzkf6z34nrhc4hkscreq7z4bqn2zhqg65e": "KLUv/SAgxQAAcAAAAAAAgFbAAD5AgFZAAwBAyArdCxwB", + "bafyr4ien3weyp7qrsanvp5dwr6b427dwjb4unrxrrxr3jsemlzuam2yksq": "iNgqWCUAAVUeIDeeOGCg4yFqF0vyS7/RRD8i9xYEf4yTU/W9fsBiTC5o2CpYJQABVR4gjPTtlRj+3DJHn/1DOUGHdxhUVrRg9pPo6ZgXH+m+z8z29vb29vY=", + "bafyr4iglueebu33cpmsrmbrl6lm65ltvoyd6wmxvlbeo7kilq6xy5kzdeu": "o2ZjaHVua3OlanNoYXJkX2NpZHOB2CpYJQABcR4gjd2Jh/4RkBtX9HaPg818dkh5RsbxjeO0yIxeaAZrCpRrYXJyYXlfc2hhcGWDBAQGa2NodW5rX3NoYXBlgwIEBm9zaGFyZGluZ19jb25maWehcGNodW5rc19wZXJfc2hhcmQIcnByaW1hcnlfYXJyYXlfcGF0aGR0ZW1waG1ldGFkYXRhqGdsYXQvYy8w2CpYJQABVR4g9ihE80+1JBvz2082VF9nfGxOLh1SFEkP54GDdZPA3ulnbG9uL2MvMNgqWCUAAVUeIIoijia5htojacxRdMbtZmbKSlugCL2qBvJmQ0af3IEeaHRpbWUvYy8w2CpYJQABVR4giTHKlrgowIlsQFAMe5EmSkM4sXZ7FlUEu1UTHPh3GpBpemFyci5qc29u2CpYJQABVR4gpwSyFzRiBbba4+sInd8eb/uNgLudRAa/I4EsCDmt6GltbGF0L3phcnIuanNvbtgqWCUAAVUeIGnYnkci1sY7Eb7/mgbVkOvZJtD/4w3qopInfoVsIs+pbWxvbi96YXJyLmpzb27YKlglAAFVHiBhqrP17aHf/A8dUW8ScpEAUe/wuPSB5obuRFqbLVUtn250ZW1wL3phcnIuanNvbtgqWCUAAVUeIE9qnu2+RIhRxpFR0sZMjAnIxazpUNApdUKy6+vVtbW4bnRpbWUvemFyci5qc29u2CpYJQABVR4gWWSizX0CrNBbirZxnRDjwN6SevI8kBa9irCpavoPimVwbWFuaWZlc3RfdmVyc2lvbm9zaGFyZGVkX3phcnJfdjE=" + }, + "root_cid": "bafyr4iglueebu33cpmsrmbrl6lm65ltvoyd6wmxvlbeo7kilq6xy5kzdeu" +} diff --git a/tests/test_converter.py b/tests/test_converter.py index 64e5799..dd3c729 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -48,7 +48,7 @@ def converter_test_dataset(): yield ds -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio async def test_converter_produces_identical_dataset( create_ipfs: tuple[str, str], converter_test_dataset: xr.Dataset, @@ -141,7 +141,7 @@ async def test_converter_produces_identical_dataset( print("=" * 80) -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio async def test_hamt_to_sharded_cli_success( create_ipfs: tuple[str, str], converter_test_dataset: xr.Dataset, capsys ): @@ -192,7 +192,7 @@ async def test_hamt_to_sharded_cli_success( xr.testing.assert_identical(test_ds, ds_from_sharded) -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio async def test_hamt_to_sharded_cli_default_args( create_ipfs: tuple[str, str], converter_test_dataset: xr.Dataset, capsys ): @@ -238,7 +238,7 @@ async def test_hamt_to_sharded_cli_default_args( xr.testing.assert_identical(test_ds, ds_from_sharded) -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio async def test_hamt_to_sharded_cli_invalid_cid(create_ipfs: tuple[str, str], capsys): """ Tests the CLI with an invalid hamt_cid. diff --git a/tests/test_h1_delete_canonicalization.py b/tests/test_h1_delete_canonicalization.py new file mode 100644 index 0000000..96848e7 --- /dev/null +++ b/tests/test_h1_delete_canonicalization.py @@ -0,0 +1,83 @@ +from typing import Literal + +import pytest +from hypothesis import example, given, settings +from hypothesis import strategies as st + +from py_hamt import HAMT, InMemoryCAS + +Operation = tuple[Literal["set", "delete"], str] + +# These numeric strings have the same first byte in their BLAKE3 hashes. The +# additional keys exercise interleavings outside that colliding root bucket. +COLLIDING_KEYS = ("5", "15", "123", "317", "349") +KEY_POOL = (*COLLIDING_KEYS, "0", "1", "2", "alpha", "omega") +DELETE_AFTER_SPLIT: list[Operation] = [ + *(("set", key) for key in COLLIDING_KEYS), + *(("delete", key) for key in COLLIDING_KEYS[1:]), +] + + +@pytest.mark.asyncio +async def test_delete_collapses_split_tree_to_canonical_root() -> None: + """Canonical deletion changes CIDs only for formerly non-canonical trees.""" + cas = InMemoryCAS() + + mutated_hamt = await HAMT.build(cas=cas, max_bucket_size=1) + await mutated_hamt.set("5", b"v") + await mutated_hamt.set("15", b"v") + await mutated_hamt.delete("15") + await mutated_hamt.make_read_only() + + fresh_hamt = await HAMT.build(cas=cas, max_bucket_size=1) + await fresh_hamt.set("5", b"v") + await fresh_hamt.make_read_only() + + assert await mutated_hamt.get("5") == await fresh_hamt.get("5") == b"v" + assert mutated_hamt.root_node_id == fresh_hamt.root_node_id, ( + "determinism violation: identical key-value content produced different " + "root node IDs after delete" + ) + + +@pytest.mark.parametrize("max_bucket_size", range(1, 5)) +@pytest.mark.asyncio +@given( + operations=st.lists( + st.tuples( + st.sampled_from(("set", "delete")), + st.sampled_from(KEY_POOL), + ), + min_size=1, + max_size=30, + ) +) +@example(operations=DELETE_AFTER_SPLIT) +@settings(max_examples=50, deadline=None, derandomize=True) +async def test_operation_history_does_not_change_root_id( + max_bucket_size: int, operations: list[Operation] +) -> None: + cas = InMemoryCAS() + mutated_hamt = await HAMT.build(cas=cas, max_bucket_size=max_bucket_size) + surviving_values: dict[str, bytes] = {} + + for operation, key in operations: + if operation == "set": + value = f"value:{key}".encode() + await mutated_hamt.set(key, value) + surviving_values[key] = value + elif key in surviving_values: + await mutated_hamt.delete(key) + del surviving_values[key] + + await mutated_hamt.make_read_only() + + fresh_hamt = await HAMT.build(cas=cas, max_bucket_size=max_bucket_size) + for key, value in sorted(surviving_values.items()): + await fresh_hamt.set(key, value) + await fresh_hamt.make_read_only() + + assert mutated_hamt.root_node_id == fresh_hamt.root_node_id, ( + "determinism violation: identical surviving key-value content produced " + f"different root node IDs for max_bucket_size={max_bucket_size}" + ) diff --git a/tests/test_h2_enable_write_idempotent.py b/tests/test_h2_enable_write_idempotent.py new file mode 100644 index 0000000..39a1d0f --- /dev/null +++ b/tests/test_h2_enable_write_idempotent.py @@ -0,0 +1,19 @@ +import pytest + +from py_hamt import HAMT, InMemoryCAS + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redundant_calls", [1, 2, 3]) +async def test_enable_write_is_idempotent_while_already_writable( + redundant_calls: int, +) -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + await hamt.set("foo", b"bar") + + for _ in range(redundant_calls): + await hamt.enable_write() + + assert await hamt.get("foo") == b"bar" + assert [key async for key in hamt.keys()] == ["foo"] + assert await hamt.len() == 1 diff --git a/tests/test_h3_failed_set_atomicity.py b/tests/test_h3_failed_set_atomicity.py new file mode 100644 index 0000000..daada98 --- /dev/null +++ b/tests/test_h3_failed_set_atomicity.py @@ -0,0 +1,24 @@ +import pytest + +from py_hamt import HAMT, InMemoryCAS + + +def one_byte_colliding_hash(_: bytes) -> bytes: + return b"\x00" + + +@pytest.mark.asyncio +async def test_failed_set_preserves_previously_committed_key() -> None: + hamt = await HAMT.build( + cas=InMemoryCAS(), + hash_fn=one_byte_colliding_hash, + max_bucket_size=1, + ) + + await hamt.set("a", b"value-a") + assert await hamt.get("a") == b"value-a" + + with pytest.raises(IndexError): + await hamt.set("b", b"value-b") + + assert await hamt.get("a") == b"value-a" diff --git a/tests/test_h4_keys_iteration_deadlock.py b/tests/test_h4_keys_iteration_deadlock.py new file mode 100644 index 0000000..b5288e2 --- /dev/null +++ b/tests/test_h4_keys_iteration_deadlock.py @@ -0,0 +1,79 @@ +import asyncio + +import pytest + +from py_hamt import HAMT, InMemoryCAS + + +@pytest.mark.asyncio +async def test_set_inside_keys_iteration_does_not_deadlock() -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + await hamt.set("existing", b"value") + + async def iterate_and_set() -> None: + async for key in hamt.keys(): + if key == "existing": + await hamt.set("added", b"new value") + + try: + await asyncio.wait_for(iterate_and_set(), timeout=5) + except TimeoutError: + pytest.fail( + "DEADLOCK: set() inside hamt.keys() iteration did not complete " + "within 5 seconds" + ) + + assert await hamt.get("existing") == b"value" + assert await hamt.get("added") == b"new value" + + +@pytest.mark.asyncio +async def test_get_inside_keys_iteration_does_not_deadlock() -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + expected_values = {"first": b"one", "second": b"two"} + for key, value in expected_values.items(): + await hamt.set(key, value) + + values_seen: dict[str, bytes] = {} + + async def iterate_and_get() -> None: + async for key in hamt.keys(): + value = await hamt.get(key) + assert isinstance(value, bytes) + values_seen[key] = value + + try: + await asyncio.wait_for(iterate_and_get(), timeout=5) + except TimeoutError: + pytest.fail( + "DEADLOCK: get() inside hamt.keys() iteration did not complete " + "within 5 seconds" + ) + + assert values_seen == expected_values + + +@pytest.mark.asyncio +async def test_keys_iteration_uses_snapshot_when_mutated_between_yields() -> None: + hamt = await HAMT.build(cas=InMemoryCAS(), max_bucket_size=1) + original_values = {f"key-{index}": f"value-{index}".encode() for index in range(20)} + for key, value in original_values.items(): + await hamt.set(key, value) + + keys_iterator = hamt.keys() + first_key = await anext(keys_iterator) + + deleted_key = next(key for key in original_values if key != first_key) + await asyncio.gather( + hamt.delete(deleted_key), + hamt.set("added", b"new value"), + ) + iterated_keys = {first_key, *[key async for key in keys_iterator]} + + assert iterated_keys == set(original_values) + + expected_values = original_values | {"added": b"new value"} + del expected_values[deleted_key] + assert set([key async for key in hamt.keys()]) == set(expected_values) + for key, expected_value in expected_values.items(): + assert await hamt.get(key) == expected_value diff --git a/tests/test_h5_read_cache_growth.py b/tests/test_h5_read_cache_growth.py new file mode 100644 index 0000000..467f83c --- /dev/null +++ b/tests/test_h5_read_cache_growth.py @@ -0,0 +1,153 @@ +from collections import defaultdict +from typing import DefaultDict, cast + +import pytest +from dag_cbor.ipld import IPLDKind + +from py_hamt import HAMT, InMemoryCAS +from py_hamt.hamt import InMemoryTreeStore, Node + + +def retained_node_count(node_store: object) -> int: + """Count Nodes retained in any dictionary owned by a node store.""" + total = 0 + for attribute in vars(node_store).values(): + if isinstance(attribute, dict) and all( + isinstance(value, Node) for value in attribute.values() + ): + total += len(attribute) + return total + + +class CountingInMemoryCAS(InMemoryCAS): + def __init__(self) -> None: + super().__init__() + self.load_calls: DefaultDict[IPLDKind, int] = defaultdict(int) + + async def load( + self, + id: IPLDKind, + offset: int | None = None, + length: int | None = None, + suffix: int | None = None, + ) -> bytes: + self.load_calls[id] += 1 + return await super().load(id, offset=offset, length=length, suffix=suffix) + + +@pytest.mark.asyncio +async def test_repeated_get_does_not_grow_buffer() -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + for index in range(20): + await hamt.set(f"key-{index}", f"value-{index}") + await hamt.cache_vacate() + + assert await hamt.get("key-0") == "value-0" + count_after_first_get = retained_node_count(hamt.node_store) + + for _ in range(50): + assert await hamt.get("key-0") == "value-0" + count_after_repeated_gets = retained_node_count(hamt.node_store) + + assert count_after_repeated_gets == count_after_first_get, ( + "repeated reads retained additional nodes: " + f"{count_after_first_get} after warm-up, " + f"{count_after_repeated_gets} after 50 more gets" + ) + + +@pytest.mark.asyncio +async def test_second_load_of_same_cas_id_hits_cache() -> None: + cas = CountingInMemoryCAS() + hamt = await HAMT.build(cas=cas) + for index in range(4): + await hamt.set(f"key-{index}", f"value-{index}") + await hamt.cache_vacate() + + root_node_id = hamt.root_node_id + calls_before_first_get = cas.load_calls[root_node_id] + assert await hamt.get("key-0") == "value-0" + first_get_loads = cas.load_calls[root_node_id] - calls_before_first_get + + calls_before_second_get = cas.load_calls[root_node_id] + assert await hamt.get("key-0") == "value-0" + second_get_loads = cas.load_calls[root_node_id] - calls_before_second_get + + assert first_get_loads > 0 + assert second_get_loads == 0, ( + "the second lookup reloaded the same root node from CAS " + f"{second_get_loads} time(s)" + ) + + +async def build_reference_hamt() -> tuple[HAMT, dict[str, str]]: + values = {f"key-{index}": f"value-{index}" for index in range(32)} + hamt = await HAMT.build(cas=InMemoryCAS()) + for key, value in values.items(): + await hamt.set(key, value) + await hamt.make_read_only() + return hamt, values + + +@pytest.mark.asyncio +async def test_vacate_root_cid_golden() -> None: + first_hamt, values = await build_reference_hamt() + second_hamt, _ = await build_reference_hamt() + + assert first_hamt.root_node_id == second_hamt.root_node_id + for key, value in values.items(): + assert await first_hamt.get(key) == value + assert await second_hamt.get(key) == value + + +@pytest.mark.asyncio +async def test_clean_cache_handles_unhashable_ids_and_contributes_to_size() -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + node_store = cast(InMemoryTreeStore, hamt.node_store) + unhashable_id = cast(IPLDKind, []) + + assert node_store.get_clean_node(unhashable_id) is None + node_store.cache_clean_node(unhashable_id, Node()) + node_store.cache_clean_node(b"existing-node", Node()) + node_store.remove_clean_node(unhashable_id) + + cached_node = Node() + node_store.cache_clean_node(b"clean-node", cached_node) + assert node_store.size() >= len(cached_node.serialize()) + + +@pytest.mark.asyncio +async def test_reserialize_prunes_an_empty_child_from_its_parent() -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + node_store = cast(InMemoryTreeStore, hamt.node_store) + root_id = hamt.root_node_id + root_node = await node_store.load(root_id) + empty_child = Node() + child_id = await node_store.add_to_buffer(empty_child) + child_slot = 7 + root_node.set_link(child_slot, child_id) + node_stack = [(root_id, root_node), (child_id, empty_child)] + link_path = [0, child_slot] + + await hamt._reserialize_and_link(node_stack, link_path) + + assert node_stack == [(root_id, root_node)] + assert link_path == [0] + assert root_node.data[child_slot] == {} + + +@pytest.mark.asyncio +async def test_collect_subtree_entries_follows_links_and_honors_limit() -> None: + cas = InMemoryCAS() + hamt = await HAMT.build(cas=cas) + child = Node() + child.data[0] = {"first": "value", "second": "value"} + child_id = await cas.save(child.serialize(), codec="dag-cbor") + parent = Node() + parent.set_link(3, child_id) + + assert await hamt._collect_subtree_entries(parent, limit=2) == { + "first": "value", + "second": "value", + } + assert await hamt._collect_subtree_entries(parent, limit=1) is None diff --git a/tests/test_h6_write_perf_restore.py b/tests/test_h6_write_perf_restore.py new file mode 100644 index 0000000..bb3add9 --- /dev/null +++ b/tests/test_h6_write_perf_restore.py @@ -0,0 +1,110 @@ +from copy import deepcopy as stdlib_deepcopy +from typing import Any + +import pytest + +from py_hamt import HAMT, InMemoryCAS +from py_hamt import hamt as hamt_module + + +def two_byte_prefix_hash(data: bytes) -> bytes: + """Force a three-node path while retaining deterministic key distribution.""" + return b"\x00\x00" + hamt_module.blake3_hashfn(data)[:30] + + +def one_byte_colliding_hash(_: bytes) -> bytes: + return b"\x00" + + +def cascading_overflow_hash(data: bytes) -> bytes: + """Collide twice before distributing keys at the third tree level.""" + return b"\x00\x00" + data + + +@pytest.mark.asyncio +async def test_happy_path_overwrites_do_not_deepcopy_the_traversal_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + hamt = await HAMT.build( + cas=InMemoryCAS(), + hash_fn=two_byte_prefix_hash, + values_are_bytes=True, + ) + key_count = 3000 + for index in range(key_count): + await hamt.set(f"key-{index}", f"value-{index}".encode()) + + deepcopy_call_count = 0 + + def counting_deepcopy(value: Any, memo: dict[int, Any] | None = None) -> Any: + nonlocal deepcopy_call_count + deepcopy_call_count += 1 + return stdlib_deepcopy(value, memo) + + monkeypatch.setattr(hamt_module, "deepcopy", counting_deepcopy) + + happy_set_count = 128 + for index in range(happy_set_count): + await hamt.set(f"key-{index}", f"updated-{index}".encode()) + + deepcopy_bound = 2 * happy_set_count + assert deepcopy_call_count <= deepcopy_bound, ( + f"happy-path overwrites made {deepcopy_call_count} deepcopy calls; " + f"expected at most {deepcopy_bound} for {happy_set_count} sets" + ) + + +@pytest.mark.asyncio +async def test_write_perf_fix_preserves_golden_root_cid() -> None: + hamt = await HAMT.build(cas=InMemoryCAS(), values_are_bytes=True) + for index in range(512): + await hamt.set(f"k{index}", f"value-{index}".encode()) + + await hamt.make_read_only() + + assert isinstance(hamt.root_node_id, bytes) + # Captured from current branch code at commit f0c2171. This guards + # serialized-output stability while the H6 write-performance fix is applied. + golden_root_cid = ( + "1e2006c31744d8396c93ff78f618082c2fd5b07bf5f3a8d328a54d14dc11cae79165" + ) + assert hamt.root_node_id.hex() == golden_root_cid + + +@pytest.mark.asyncio +async def test_detached_subtree_cascading_overflow_preserves_all_values() -> None: + hamt = await HAMT.build( + cas=InMemoryCAS(), + hash_fn=cascading_overflow_hash, + max_bucket_size=1, + values_are_bytes=True, + ) + expected_values = {"a": b"value-a", "b": b"value-b"} + + for key, value in expected_values.items(): + await hamt.set(key, value) + + for key, value in expected_values.items(): + assert await hamt.get(key) == value + + +@pytest.mark.asyncio +async def test_failed_bucket_reflow_remains_atomic_and_writable() -> None: + hamt = await HAMT.build( + cas=InMemoryCAS(), + hash_fn=one_byte_colliding_hash, + max_bucket_size=2, + values_are_bytes=True, + ) + committed_values = {"a": b"value-a", "b": b"value-b"} + for key, value in committed_values.items(): + await hamt.set(key, value) + + with pytest.raises(IndexError): + await hamt.set("c", b"value-c") + + for key, value in committed_values.items(): + assert await hamt.get(key) == value + + await hamt.set("a", b"updated-a") + assert await hamt.get("a") == b"updated-a" diff --git a/tests/test_h7_delete_failure_atomicity.py b/tests/test_h7_delete_failure_atomicity.py new file mode 100644 index 0000000..9fc2819 --- /dev/null +++ b/tests/test_h7_delete_failure_atomicity.py @@ -0,0 +1,90 @@ +from typing import Optional + +import pytest +from dag_cbor.ipld import IPLDKind + +from py_hamt import HAMT, InMemoryCAS + +# These keys share the first TWO blake3 hash bytes, forcing a depth-2 subtree; +# "s45" shares only the first byte, so it sits in a bucket of the depth-1 node +# whose collapse must load the sibling subtree from the CAS. +DEEP_KEYS = ("1313", "5428", "11835", "17462", "20814") +SHALLOW_KEY = "s45" +ALL_KEYS = (*DEEP_KEYS, SHALLOW_KEY) + + +class FlakyCAS(InMemoryCAS): + """Fails the Nth load after arming, so tests can sweep every load index a + delete performs — from the root load through the collapse sibling loads.""" + + def __init__(self) -> None: + super().__init__() + self.armed = False + self.loads_before_failure = 0 + + async def load( + self, + node_id: IPLDKind, + offset: Optional[int] = None, + length: Optional[int] = None, + suffix: Optional[int] = None, + ) -> bytes: + if self.armed: + if self.loads_before_failure <= 0: + raise ConnectionError("simulated CAS load failure") + self.loads_before_failure -= 1 + return await super().load(node_id, offset=offset, length=length, suffix=suffix) + + +async def _vacated_hamt(cas: FlakyCAS) -> HAMT: + hamt = await HAMT.build(cas=cas) + for key in ALL_KEYS: + await hamt.set(key, key.encode()) + await hamt.cache_vacate() + return hamt + + +@pytest.mark.asyncio +async def test_failed_delete_preserves_all_keys() -> None: + """The collapse-phase CAS failure: walk loads are warmed into the cache, so + the first load delete performs is the sibling-subtree load that runs after + the bucket mutation in the pre-fix code.""" + cas = FlakyCAS() + hamt = await _vacated_hamt(cas) + assert await hamt.get(SHALLOW_KEY) == SHALLOW_KEY.encode() + + cas.armed = True + with pytest.raises(ConnectionError): + await hamt.delete(SHALLOW_KEY) + cas.armed = False + + for key in ALL_KEYS: + assert await hamt.get(key) == key.encode() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("loads_before_failure", range(6)) +async def test_delete_is_atomic_at_every_load_index( + loads_before_failure: int, +) -> None: + """Sweep the failure across every CAS load a cold delete performs — the + walk loads (pre-mutation) and the collapse prefetch loads alike. Whenever + delete raises, every key must remain readable; once the budget exceeds the + loads a delete needs, the delete must succeed.""" + cas = FlakyCAS() + hamt = await _vacated_hamt(cas) + + cas.armed = True + cas.loads_before_failure = loads_before_failure + try: + await hamt.delete(SHALLOW_KEY) + except ConnectionError: + cas.armed = False + for key in ALL_KEYS: + assert await hamt.get(key) == key.encode() + else: + cas.armed = False + with pytest.raises(KeyError): + await hamt.get(SHALLOW_KEY) + for key in DEEP_KEYS: + assert await hamt.get(key) == key.encode() diff --git a/tests/test_hamt.py b/tests/test_hamt.py index de8c32c..3d3748b 100644 --- a/tests/test_hamt.py +++ b/tests/test_hamt.py @@ -133,9 +133,11 @@ async def get_and_vacate(k: str, v: IPLDKind) -> None: assert (await read_hamt.get(k)) == v if (await read_hamt.cache_size()) > small_cache_size_bytes: await read_hamt.cache_vacate() - assert (await read_hamt.cache_size()) == 0 await asyncio.gather(*[get_and_vacate(k, v) for k, v in kvs]) + # A sibling coroutine may repopulate the cache after another vacate returns. + await read_hamt.cache_vacate() + assert (await read_hamt.cache_size()) == 0 # In memory tree while writing # set small max bucket size to force more linking and more nodes @@ -146,10 +148,12 @@ async def set_and_vacate(k: str, v: IPLDKind) -> None: assert (await small_memory_tree.get(k)) == v if (await small_memory_tree.cache_size()) > small_cache_size_bytes: await small_memory_tree.cache_vacate() - assert (await small_memory_tree.cache_size()) == 0 assert (await small_memory_tree.get(k)) == v await asyncio.gather(*[set_and_vacate(k, v) for k, v in kvs]) + # Check cache emptiness only after concurrent writers have finished. + await small_memory_tree.cache_vacate() + assert (await small_memory_tree.cache_size()) == 0 @pytest.mark.asyncio diff --git a/tests/test_k10_http2_enable.py b/tests/test_k10_http2_enable.py new file mode 100644 index 0000000..2cdc84b --- /dev/null +++ b/tests/test_k10_http2_enable.py @@ -0,0 +1,229 @@ +import asyncio +import ssl +from collections.abc import Iterator +from dataclasses import dataclass, field +from pathlib import Path +from threading import Event, Thread +from typing import cast + +import pytest +import trustme +from h2 import events +from h2.config import H2Configuration +from h2.connection import H2Connection +from multiformats import CID + +from py_hamt import KuboCAS + +EXPECTED_BODY = b"gateway response across HTTP/2 GOAWAY" +TEST_CID = CID.decode("bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad7lrm") +MAX_STREAMS = 4 +CONCURRENT_LOADS = 12 +GOAWAY_CLOSE_DELAY = 0.05 + + +@dataclass +class TLSTestGateway: + url: str + active_transports: set[asyncio.WriteTransport] = field(default_factory=set) + + +class GoAwayGatewayProtocol(asyncio.Protocol): + """Serve HTTP/1.1 or raw HTTP/2 according to the negotiated ALPN protocol.""" + + def __init__(self, gateway: TLSTestGateway) -> None: + self.gateway = gateway + self.transport: asyncio.WriteTransport | None = None + self.http1_buffer = bytearray() + self.http1_response_sent = False + self.h2_connection = H2Connection( + config=H2Configuration(client_side=False, header_encoding="utf-8") + ) + self.h2_streams_served = 0 + self.goaway_sent = False + self.negotiated_protocol: str | None = None + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + self.transport = cast(asyncio.WriteTransport, transport) + self.gateway.active_transports.add(self.transport) + ssl_object = cast(ssl.SSLObject, transport.get_extra_info("ssl_object")) + self.negotiated_protocol = ssl_object.selected_alpn_protocol() + + if self.negotiated_protocol == "h2": + self.h2_connection.initiate_connection() + self.transport.write(self.h2_connection.data_to_send()) + + def data_received(self, data: bytes) -> None: + if self.negotiated_protocol == "h2": + self._receive_http2(data) + else: + self._receive_http1(data) + + def connection_lost(self, exc: Exception | None) -> None: + if self.transport is not None: + self.gateway.active_transports.discard(self.transport) + self.transport = None + + def _receive_http1(self, data: bytes) -> None: + if self.transport is None or self.http1_response_sent: + return + self.http1_buffer.extend(data) + if b"\r\n\r\n" not in self.http1_buffer: + return + + self.http1_response_sent = True + response = ( + b"HTTP/1.1 200 OK\r\n" + + f"Content-Length: {len(EXPECTED_BODY)}\r\n".encode() + + b"Content-Type: application/octet-stream\r\n" + + b"Connection: close\r\n\r\n" + + EXPECTED_BODY + ) + self.transport.write(response) + self.transport.close() + + def _receive_http2(self, data: bytes) -> None: + if self.transport is None: + return + try: + received_events = self.h2_connection.receive_data(data) + except Exception: + # Protocol callbacks must close the connection on any decoder + # failure rather than let it escape into the event loop. + self.transport.close() + return + + for event in received_events: + if isinstance(event, events.RequestReceived): + self._serve_http2_stream(event.stream_id) + + pending_data = self.h2_connection.data_to_send() + if pending_data: + self.transport.write(pending_data) + + def _serve_http2_stream(self, stream_id: int) -> None: + if self.goaway_sent: + return + + self.h2_streams_served += 1 + self.h2_connection.send_headers( + stream_id, + [ + (":status", "200"), + ("content-length", str(len(EXPECTED_BODY))), + ("content-type", "application/octet-stream"), + ], + ) + self.h2_connection.send_data(stream_id, EXPECTED_BODY, end_stream=True) + + if self.h2_streams_served >= MAX_STREAMS: + self.goaway_sent = True + self.h2_connection.close_connection(last_stream_id=stream_id) + asyncio.get_running_loop().call_later( + GOAWAY_CLOSE_DELAY, self._close_transport + ) + + def _close_transport(self) -> None: + if self.transport is not None: + self.transport.close() + + +@pytest.fixture +def tls_goaway_gateway( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[TLSTestGateway]: + ca = trustme.CA() + server_certificate = ca.issue_cert("127.0.0.1") + ca_path = tmp_path / "ca.pem" + ca.cert_pem.write_to_path(ca_path) + monkeypatch.setenv("SSL_CERT_FILE", str(ca_path)) + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_certificate.configure_cert(ssl_context) + ssl_context.set_alpn_protocols(["h2", "http/1.1"]) + + gateway = TLSTestGateway(url="") + ready = Event() + server_loop = asyncio.new_event_loop() + setup_errors: list[BaseException] = [] + + def run_server() -> None: + asyncio.set_event_loop(server_loop) + server: asyncio.Server | None = None + try: + server = server_loop.run_until_complete( + server_loop.create_server( + lambda: GoAwayGatewayProtocol(gateway), + host="127.0.0.1", + port=0, + ssl=ssl_context, + ) + ) + port = server.sockets[0].getsockname()[1] + gateway.url = f"https://127.0.0.1:{port}" + ready.set() + server_loop.run_forever() + except Exception as error: + setup_errors.append(error) + ready.set() + finally: + if server is not None: + server.close() + server_loop.run_until_complete(server.wait_closed()) + for transport in list(gateway.active_transports): + transport.close() + server_loop.run_until_complete(asyncio.sleep(0)) + server_loop.close() + + server_thread = Thread(target=run_server, daemon=True) + server_thread.start() + if not ready.wait(timeout=5): + raise RuntimeError("TLS test server did not start") + if setup_errors: + raise RuntimeError("TLS test server failed to start") from setup_errors[0] + + try: + yield gateway + finally: + server_loop.call_soon_threadsafe(server_loop.stop) + server_thread.join(timeout=5) + if server_thread.is_alive(): + raise RuntimeError("TLS test server did not stop") + + +@pytest.mark.asyncio +async def test_internal_client_negotiates_http2( + tls_goaway_gateway: TLSTestGateway, +) -> None: + cas = KuboCAS( + gateway_base_url=tls_goaway_gateway.url, + rpc_base_url="http://127.0.0.1:1", + ) + try: + client = cas._loop_client() + response = await client.get(f"{tls_goaway_gateway.url}/ipfs/{TEST_CID}") + assert response.http_version == "HTTP/2" + finally: + await cas.aclose() + + +@pytest.mark.asyncio +async def test_concurrent_loads_survive_http2_goaway( + tls_goaway_gateway: TLSTestGateway, +) -> None: + cas = KuboCAS( + gateway_base_url=tls_goaway_gateway.url, + rpc_base_url="http://127.0.0.1:1", + max_retries=4, + initial_delay=0.05, + backoff_factor=1.5, + ) + try: + results = await asyncio.gather( + *(cas.load(TEST_CID) for _ in range(CONCURRENT_LOADS)) + ) + finally: + await cas.aclose() + + assert results == [EXPECTED_BODY] * CONCURRENT_LOADS diff --git a/tests/test_k12_cross_loop_client_policy.py b/tests/test_k12_cross_loop_client_policy.py new file mode 100644 index 0000000..e446f07 --- /dev/null +++ b/tests/test_k12_cross_loop_client_policy.py @@ -0,0 +1,120 @@ +import asyncio +import warnings +from collections.abc import Iterator + +import httpx +import pytest + +from py_hamt import KuboCAS + + +@pytest.fixture(autouse=True) +def preserve_current_event_loop() -> Iterator[None]: + """Restore the current loop after tests that use ``asyncio.run``.""" + try: + previous_loop = asyncio.get_event_loop() + except RuntimeError: + previous_loop = None + + try: + yield + finally: + if previous_loop is not None: + asyncio.set_event_loop(previous_loop) + + +def _client_for_current_loop(cas: KuboCAS) -> httpx.AsyncClient: + async def get_client() -> httpx.AsyncClient: + return cas._loop_client() + + return asyncio.run(get_client()) + + +def _close_clients(cas: KuboCAS, *clients: httpx.AsyncClient) -> None: + async def close_clients() -> None: + for client in {*cas._client_per_loop.values(), *clients}: + if not client.is_closed: + await client.aclose() + + asyncio.run(close_clients()) + + +def test_client_factory_builds_per_loop_clients() -> None: + factory_calls = 0 + created_clients: list[httpx.AsyncClient] = [] + + def client_factory() -> httpx.AsyncClient: + nonlocal factory_calls + factory_calls += 1 + client = httpx.AsyncClient(headers={"X-Marker": "factory"}) + created_clients.append(client) + return client + + cas = KuboCAS(client_factory=client_factory) + + try: + first_client = _client_for_current_loop(cas) + second_client = _client_for_current_loop(cas) + + assert factory_calls == 2 + assert first_client.headers["X-Marker"] == "factory" + assert second_client.headers["X-Marker"] == "factory" + + asyncio.run(cas.aclose()) + + assert all(client.is_closed for client in created_clients) + finally: + _close_clients(cas, *created_clients) + + +def test_client_and_factory_are_mutually_exclusive() -> None: + supplied_client = httpx.AsyncClient() + + try: + with pytest.raises(ValueError): + KuboCAS( + client=supplied_client, + client_factory=lambda: httpx.AsyncClient(), + ) + finally: + asyncio.run(supplied_client.aclose()) + + +def test_second_loop_fallback_warns() -> None: + supplied_client = httpx.AsyncClient() + cas = KuboCAS(client=supplied_client) + + try: + assert _client_for_current_loop(cas) is supplied_client + + with pytest.warns( + RuntimeWarning, + match="cannot be reused across event loops", + ): + _client_for_current_loop(cas) + finally: + _close_clients(cas, supplied_client) + + +def test_second_loop_fallback_preserves_redirect_policy() -> None: + supplied_client = httpx.AsyncClient(follow_redirects=False) + cas = KuboCAS(client=supplied_client) + + try: + assert _client_for_current_loop(cas) is supplied_client + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) + fallback_client = _client_for_current_loop(cas) + + assert fallback_client.follow_redirects is False + finally: + _close_clients(cas, supplied_client) + + +def test_client_factory_rejects_headers_and_auth() -> None: + with pytest.raises(ValueError, match="mutually exclusive with headers/auth"): + KuboCAS( + client_factory=httpx.AsyncClient, + headers={"X-Header": "value"}, + ) diff --git a/tests/test_k13_per_loop_aclose.py b/tests/test_k13_per_loop_aclose.py new file mode 100644 index 0000000..925cbb7 --- /dev/null +++ b/tests/test_k13_per_loop_aclose.py @@ -0,0 +1,337 @@ +import asyncio +import queue +import threading +import warnings +from collections.abc import Iterator + +import httpx +import pytest + +import py_hamt.store_httpx as store_httpx +from py_hamt import KuboCAS + + +@pytest.fixture(autouse=True) +def preserve_current_event_loop() -> Iterator[None]: + """Restore the current loop after tests that use ``asyncio.run``.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + try: + previous_loop = asyncio.get_event_loop() + except RuntimeError: + previous_loop = None + + try: + yield + finally: + if previous_loop is not None: + asyncio.set_event_loop(previous_loop) + + +class LoopBoundRecordingTransport(httpx.AsyncBaseTransport): + """Record whether cleanup reached the transport on its owning loop.""" + + def __init__(self) -> None: + self.owner_loop = asyncio.get_running_loop() + self.aclose_calls = 0 + self.close_calls = 0 + self.closed = False + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + raise AssertionError(f"Unexpected request: {request.method} {request.url}") + + async def aclose(self) -> None: + self.aclose_calls += 1 + if asyncio.get_running_loop() is not self.owner_loop: + raise RuntimeError("transport closed from a foreign event loop") + self.closed = True + + def close(self) -> None: + """Provide an observable synchronous fallback for a dead owner loop.""" + self.close_calls += 1 + self.closed = True + + +class UncloseableTransport(httpx.AsyncBaseTransport): + """Model a transport for which cleanup genuinely cannot complete.""" + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + raise AssertionError(f"Unexpected request: {request.method} {request.url}") + + async def aclose(self) -> None: + raise RuntimeError("transport cleanup failed") + + +def test_aclose_releases_transports_owned_by_closed_loops() -> None: + """Sequential ``asyncio.run`` clients must not lose their transports.""" + transports: list[LoopBoundRecordingTransport] = [] + clients: list[httpx.AsyncClient] = [] + + def client_factory() -> httpx.AsyncClient: + transport = LoopBoundRecordingTransport() + client = httpx.AsyncClient(transport=transport) + transports.append(transport) + clients.append(client) + return client + + cas = KuboCAS(client_factory=client_factory) + + async def create_client() -> None: + cas._loop_client() + + asyncio.run(create_client()) + asyncio.run(create_client()) + + assert len(transports) == 2 + assert all(transport.owner_loop.is_closed() for transport in transports) + + asyncio.run(cas.aclose()) + + # AsyncClient marks itself closed before awaiting its transport, so the + # transport state is the resource-release assertion that matters here. + assert all(client.is_closed for client in clients) + assert [transport.closed for transport in transports] == [True, True] + + +def test_aclose_warns_for_stock_transport_owned_by_closed_loop() -> None: + """A stock transport on a dead loop degrades to warned best-effort cleanup.""" + server_ports: queue.Queue[int] = queue.Queue() + server_errors: queue.Queue[BaseException] = queue.Queue() + stop_server = threading.Event() + + async def handle_request( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + try: + await reader.readuntil(b"\r\n\r\n") + writer.write( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 2\r\n" + b"Connection: keep-alive\r\n" + b"\r\n" + b"OK" + ) + await writer.drain() + await reader.read() + finally: + writer.close() + try: + await writer.wait_closed() + except ConnectionError: + pass + + async def serve() -> None: + server = await asyncio.start_server(handle_request, "127.0.0.1", 0) + assert server.sockets is not None + server_ports.put(server.sockets[0].getsockname()[1]) + try: + while not stop_server.is_set(): + await asyncio.sleep(0.01) + finally: + server.close() + await server.wait_closed() + + def run_server() -> None: + try: + asyncio.run(serve()) + except BaseException as exc: + server_errors.put(exc) + + server_thread = threading.Thread(target=run_server) + server_thread.start() + server_port = server_ports.get(timeout=5) + cas = KuboCAS() + + async def make_keep_alive_request() -> None: + response = await cas._loop_client().get(f"http://127.0.0.1:{server_port}/") + assert response.status_code == 200 + + try: + asyncio.run(make_keep_alive_request()) + owner_loop = next(iter(cas._client_per_loop)) + assert owner_loop.is_closed() + + with pytest.warns( + RuntimeWarning, + match="Failed to close an internally created HTTP client", + ): + asyncio.run(cas.aclose()) + + assert cas._client_per_loop == {} + assert cas._closed + finally: + stop_server.set() + server_thread.join(timeout=5) + + assert not server_thread.is_alive() + if not server_errors.empty(): + raise server_errors.get() + + +def test_aclose_reports_a_transport_cleanup_failure() -> None: + """An unrecoverable close failure must surface the specific cleanup warning.""" + cas = KuboCAS( + client_factory=lambda: httpx.AsyncClient(transport=UncloseableTransport()) + ) + + async def create_and_close() -> None: + cas._loop_client() + await cas.aclose() + + with pytest.warns( + RuntimeWarning, + match="Failed to close an internally created HTTP client", + ): + asyncio.run(create_and_close()) + + +def test_aclose_awaits_current_loop_transport_normally() -> None: + """A transport owned by the active loop keeps the normal async path.""" + transports: list[LoopBoundRecordingTransport] = [] + + def client_factory() -> httpx.AsyncClient: + transport = LoopBoundRecordingTransport() + transports.append(transport) + return httpx.AsyncClient(transport=transport) + + cas = KuboCAS(client_factory=client_factory) + + async def create_and_close() -> None: + cas._loop_client() + await cas.aclose() + + asyncio.run(create_and_close()) + + assert len(transports) == 1 + assert transports[0].closed + assert transports[0].aclose_calls == 1 + assert transports[0].close_calls == 0 + + +def test_aclose_runs_cleanup_on_stopped_foreign_owner_loop() -> None: + """A stopped foreign owner loop is temporarily run for cleanup.""" + transports: list[LoopBoundRecordingTransport] = [] + + def client_factory() -> httpx.AsyncClient: + transport = LoopBoundRecordingTransport() + transports.append(transport) + return httpx.AsyncClient(transport=transport) + + cas = KuboCAS(client_factory=client_factory) + owner_loop = asyncio.new_event_loop() + + async def create_client() -> None: + cas._loop_client() + + try: + owner_loop.run_until_complete(create_client()) + + asyncio.run(cas.aclose()) + + assert len(transports) == 1 + assert transports[0].closed + assert cas._client_per_loop == {} + assert cas._closed + finally: + owner_loop.close() + + +def test_aclose_schedules_cleanup_on_running_foreign_loop() -> None: + """A live foreign loop must execute transport cleanup on that loop.""" + transports: list[LoopBoundRecordingTransport] = [] + + def client_factory() -> httpx.AsyncClient: + transport = LoopBoundRecordingTransport() + transports.append(transport) + return httpx.AsyncClient(transport=transport) + + cas = KuboCAS(client_factory=client_factory) + owner_loop = asyncio.new_event_loop() + loop_started = threading.Event() + + def run_owner_loop() -> None: + asyncio.set_event_loop(owner_loop) + loop_started.set() + owner_loop.run_forever() + + owner_thread = threading.Thread(target=run_owner_loop) + owner_thread.start() + assert loop_started.wait(timeout=5) + + async def create_client() -> None: + cas._loop_client() + + try: + create_future = asyncio.run_coroutine_threadsafe(create_client(), owner_loop) + create_future.result(timeout=5) + + asyncio.run(cas.aclose()) + + assert len(transports) == 1 + assert transports[0].closed + assert transports[0].aclose_calls == 1 + assert cas._client_per_loop == {} + assert cas._closed + finally: + owner_loop.call_soon_threadsafe(owner_loop.stop) + owner_thread.join(timeout=5) + assert not owner_thread.is_alive() + owner_loop.close() + + +def test_aclose_falls_back_when_running_owner_loop_stalls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A running owner loop that stalls mid-close must not hang aclose(). + + The loop passes the ``is_running()`` check but then never executes the + scheduled close, so ``wrap_future`` would wait forever. The bounded wait + times out, cancels the scheduled close, and falls back to the synchronous + transport shutdown. + """ + monkeypatch.setattr(store_httpx, "_CROSS_LOOP_ACLOSE_TIMEOUT_S", 0.05) + + transports: list[LoopBoundRecordingTransport] = [] + + def client_factory() -> httpx.AsyncClient: + transport = LoopBoundRecordingTransport() + transports.append(transport) + return httpx.AsyncClient(transport=transport) + + cas = KuboCAS(client_factory=client_factory) + owner_loop = asyncio.new_event_loop() + loop_started = threading.Event() + + def run_owner_loop() -> None: + asyncio.set_event_loop(owner_loop) + loop_started.set() + owner_loop.run_forever() + + owner_thread = threading.Thread(target=run_owner_loop) + owner_thread.start() + assert loop_started.wait(timeout=5) + + async def create_client() -> None: + cas._loop_client() + + block = threading.Event() + try: + create_future = asyncio.run_coroutine_threadsafe(create_client(), owner_loop) + create_future.result(timeout=5) + + # Freeze the still-"running" owner loop so the scheduled close can never + # execute, forcing the bounded wait to time out. + owner_loop.call_soon_threadsafe(block.wait) + + asyncio.run(cas.aclose()) + + assert transports[0].close_calls == 1 + assert transports[0].closed + assert cas._client_per_loop == {} + assert cas._closed + finally: + block.set() + owner_loop.call_soon_threadsafe(owner_loop.stop) + owner_thread.join(timeout=5) + assert not owner_thread.is_alive() + owner_loop.close() diff --git a/tests/test_k14_concurrent_client_binding.py b/tests/test_k14_concurrent_client_binding.py new file mode 100644 index 0000000..3d14343 --- /dev/null +++ b/tests/test_k14_concurrent_client_binding.py @@ -0,0 +1,97 @@ +import asyncio +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Barrier, Thread + +import httpx +import pytest + +from py_hamt import KuboCAS + + +@pytest.fixture +def gateway() -> Iterator[str]: + """Offline gateway that answers every request with a small 200 body.""" + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = b"ok" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, message_format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server_thread = Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + server_thread.join() + + +def test_supplied_client_bound_to_one_loop_under_concurrent_first_use( + gateway: str, +) -> None: + """Two loops racing on first use must not bind one client to both. + + Before the first-use lock, two event loops on different threads could each + observe ``_supplied_client`` as non-None and bind the same + ``httpx.AsyncClient`` to both loops, which fails at request time with a + "bound to a different event loop" error. The lock serializes consumption so + the supplied client is used by exactly one loop and the other loop falls + back to a distinct internally created client. + """ + supplied_client = httpx.AsyncClient() + cas = KuboCAS( + client=supplied_client, + gateway_base_url=gateway, + rpc_base_url=gateway, + ) + + barrier = Barrier(2) + errors: list[BaseException] = [] + + def worker(cid: str) -> None: + async def run() -> None: + barrier.wait() # release both threads into first-use together + await cas.load(cid) + + try: + asyncio.run(run()) + except Exception as exc: # pragma: no cover - only trips on regression + errors.append(exc) + + threads = [Thread(target=worker, args=(f"cid-{i}",)) for i in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + try: + assert not errors, f"concurrent first-use raised: {errors!r}" + + bound_to_supplied = [ + loop + for loop, client in cas._client_per_loop.items() + if client is supplied_client + ] + assert len(bound_to_supplied) == 1, ( + "supplied client should be bound to exactly one loop, " + f"got {len(bound_to_supplied)}" + ) + assert cas._supplied_client is None + assert len(cas._client_per_loop) == 2 + finally: + + async def cleanup() -> None: + for client in {*cas._client_per_loop.values(), supplied_client}: + if not client.is_closed: + await client.aclose() + + asyncio.run(cleanup()) diff --git a/tests/test_k1_lazy_client_binding.py b/tests/test_k1_lazy_client_binding.py new file mode 100644 index 0000000..19ab3f5 --- /dev/null +++ b/tests/test_k1_lazy_client_binding.py @@ -0,0 +1,44 @@ +import asyncio + +import httpx +from multiformats import CID + +from py_hamt import KuboCAS + + +def test_supplied_client_is_bound_lazily_outside_event_loop() -> None: + """A supplied client can be constructed synchronously and used later.""" + expected_body = b"loaded through the supplied client" + requests: list[httpx.Request] = [] + + def handle_request(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, content=expected_body, request=request) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handle_request)) + loop = asyncio.new_event_loop() + try: + cas = KuboCAS( + client=client, + gateway_base_url="http://127.0.0.1:1", + rpc_base_url="http://127.0.0.1:1", + ) + + async def load_and_close() -> bytes: + try: + cid = CID.decode( + "bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad7lrm" + ) + return await cas.load(cid) + finally: + await client.aclose() + + loaded = loop.run_until_complete(load_and_close()) + finally: + if not client.is_closed: + loop.run_until_complete(client.aclose()) + loop.close() + + assert loaded == expected_body + assert len(requests) == 1 + assert requests[0].method == "GET" diff --git a/tests/test_k2_user_client_multiloop.py b/tests/test_k2_user_client_multiloop.py new file mode 100644 index 0000000..b0e8852 --- /dev/null +++ b/tests/test_k2_user_client_multiloop.py @@ -0,0 +1,130 @@ +import asyncio +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from typing import TypeAlias + +import httpx +import pytest + +from py_hamt import KuboCAS + +RecordedHeaders: TypeAlias = dict[str, dict[str, str]] + + +@pytest.fixture(autouse=True) +def preserve_current_event_loop() -> Iterator[None]: + """Restore the current loop after tests that use ``asyncio.run``.""" + try: + previous_loop = asyncio.get_event_loop() + except RuntimeError: + previous_loop = None + + try: + yield + finally: + if previous_loop is not None: + asyncio.set_event_loop(previous_loop) + + +@pytest.fixture +def recording_gateway() -> Iterator[tuple[str, RecordedHeaders]]: + """Run an offline HTTP gateway that records headers by requested CID.""" + recorded_headers: RecordedHeaders = {} + + class RecordingHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + cid = self.path.removeprefix("/ipfs/") + recorded_headers[cid] = dict(self.headers.items()) + body = b"gateway response" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, message_format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), RecordingHandler) + server_thread = Thread(target=server.serve_forever, daemon=True) + server_thread.start() + port = server.server_address[1] + + try: + yield f"http://127.0.0.1:{port}", recorded_headers + finally: + server.shutdown() + server.server_close() + server_thread.join() + + +def _build_cas_on_first_loop(gateway_url: str) -> tuple[KuboCAS, httpx.AsyncClient]: + cas_holder: list[KuboCAS] = [] + client_holder: list[httpx.AsyncClient] = [] + + async def phase_one() -> None: + supplied_client = httpx.AsyncClient(headers={"Authorization": "Bearer secret"}) + cas = KuboCAS( + client=supplied_client, + gateway_base_url=gateway_url, + rpc_base_url=gateway_url, + ) + await cas.load("auth1") + cas_holder.append(cas) + client_holder.append(supplied_client) + + asyncio.run(phase_one()) + return cas_holder[0], client_holder[0] + + +def _close_all_clients(cas: KuboCAS, supplied_client: httpx.AsyncClient) -> None: + async def close_clients() -> None: + for client in {*cas._client_per_loop.values(), supplied_client}: + if not client.is_closed: + await client.aclose() + + asyncio.run(close_clients()) + + +def test_user_client_auth_is_propagated_to_second_loop( + recording_gateway: tuple[str, RecordedHeaders], +) -> None: + gateway_url, recorded_headers = recording_gateway + cas, supplied_client = _build_cas_on_first_loop(gateway_url) + + try: + asyncio.run(cas.load("auth2")) + + assert recorded_headers["auth1"].get("Authorization") == "Bearer secret" + assert recorded_headers["auth2"].get("Authorization") == "Bearer secret" + finally: + _close_all_clients(cas, supplied_client) + + +def test_aclose_closes_only_internally_created_second_loop_client( + recording_gateway: tuple[str, RecordedHeaders], +) -> None: + gateway_url, _ = recording_gateway + cas, supplied_client = _build_cas_on_first_loop(gateway_url) + + internally_created_clients: list[httpx.AsyncClient] = [] + + async def phase_two() -> None: + await cas.load("auth2") + # Snapshot internally-created clients BEFORE aclose(), since a + # correct aclose() implementation may clear the per-loop mapping. + internally_created_clients.extend( + client + for client in cas._client_per_loop.values() + if client is not supplied_client + ) + await cas.aclose() + + try: + asyncio.run(phase_two()) + + assert not supplied_client.is_closed + assert internally_created_clients + assert all(client.is_closed for client in internally_created_clients) + finally: + _close_all_clients(cas, supplied_client) diff --git a/tests/test_k3_per_loop_semaphore.py b/tests/test_k3_per_loop_semaphore.py new file mode 100644 index 0000000..b701fa1 --- /dev/null +++ b/tests/test_k3_per_loop_semaphore.py @@ -0,0 +1,63 @@ +import asyncio +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from py_hamt import KuboCAS + + +def test_contended_loads_work_across_sequential_event_loops() -> None: + try: + previous_loop = asyncio.get_event_loop() + except RuntimeError: + previous_loop = None + + expected_body = b"loaded from the mock gateway" + request_started = threading.Event() + release_response = threading.Event() + + class GatewayHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + request_started.set() + release_response.wait() + self.send_response(200) + self.send_header("Content-Length", str(len(expected_body))) + self.end_headers() + self.wfile.write(expected_body) + + def log_message(self, message_format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), GatewayHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + cas = KuboCAS( + gateway_base_url=base_url, + rpc_base_url=base_url, + concurrency=1, + max_retries=0, + ) + + async def run_contended_loads() -> list[bytes]: + request_started.clear() + release_response.clear() + + first_load = asyncio.create_task(cas.load("plain")) + await asyncio.to_thread(request_started.wait) + second_load = asyncio.create_task(cas.load("plain")) + await asyncio.sleep(0) + release_response.set() + + return list(await asyncio.gather(first_load, second_load)) + + try: + assert asyncio.run(run_contended_loads()) == [expected_body, expected_body] + assert asyncio.run(run_contended_loads()) == [expected_body, expected_body] + finally: + release_response.set() + asyncio.run(cas.aclose()) + server.shutdown() + server.server_close() + server_thread.join() + if previous_loop is not None: + asyncio.set_event_loop(previous_loop) diff --git a/tests/test_k4_range_correctness.py b/tests/test_k4_range_correctness.py new file mode 100644 index 0000000..11c6112 --- /dev/null +++ b/tests/test_k4_range_correctness.py @@ -0,0 +1,392 @@ +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from typing import TypeAlias + +import httpx +import pytest + +from py_hamt import InMemoryCAS, KuboCAS + +BODY = bytes(range(100)) +RecordedRanges: TypeAlias = dict[str, list[str | None]] + + +@pytest.fixture +def range_gateway() -> Iterator[tuple[str, RecordedRanges]]: + """Run an offline gateway that can honor or ignore Range requests.""" + recorded_ranges: RecordedRanges = {} + + class RangeHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + cid = self.path.rsplit("/", 1)[-1].split("?", 1)[0] + range_header = self.headers.get("Range") + recorded_ranges.setdefault(cid, []).append(range_header) + + if ( + cid in ("range-not-satisfiable", "malformed-416") + and range_header is not None + ): + # Compliant gateway rejects a range starting at/past EOF with + # 416 + ``Content-Range: bytes */N``. ``malformed-416`` omits + # the header to exercise the non-empty-slice fallback. + self.send_response(416) + if cid == "range-not-satisfiable": + self.send_header("Content-Range", f"bytes */{len(BODY)}") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if cid == "empty-suffix-416" and range_header is not None: + # Zero-length object: a suffix range is unsatisfiable, so a + # compliant gateway answers 416 + ``Content-Range: bytes */0``. + self.send_response(416) + self.send_header("Content-Range", "bytes */0") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if cid == "unexpected-2xx" and range_header is not None: + # A non-206/200 success (203) carrying the full body: the byte + # window is unknowable, so KuboCAS must reject it. + self.send_response(203) + self.send_header("Content-Length", str(len(BODY))) + self.end_headers() + self.wfile.write(BODY) + return + + if cid.startswith("bad-206") and range_header is not None: + # A 206 whose Content-Range is absent, garbled, or inconsistent + # with the body/request. httpx's raise_for_status accepts these, + # so KuboCAS must validate them itself. + self.send_response(206) + if cid == "bad-206-no-range": + body = BODY[5:15] # no Content-Range header at all + elif cid == "bad-206-star-total": + body = BODY[5:15] + self.send_header("Content-Range", "bytes 5-14/*") + elif cid == "bad-206-bad-length": + body = BODY[5:10] # 5 bytes... + self.send_header("Content-Range", "bytes 5-14/100") # claims 10 + else: # bad-206-wrong-start + body = BODY[0:10] # consistent window, but wrong start + self.send_header("Content-Range", "bytes 0-9/100") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + + if cid in ("honors-range", "honors-suffix") and range_header is not None: + unit, requested_range = range_header.split("=", 1) + start_text, end_text = requested_range.split("-", 1) + assert unit == "bytes" + if cid == "honors-suffix": + # bytes=-N maps to the last N bytes of the object. + start = len(BODY) - int(end_text) + end = len(BODY) - 1 + else: + start = int(start_text) + # bytes=start- (open-ended) runs to the end of the object. + end = int(end_text) if end_text else len(BODY) - 1 + response_body = BODY[start : end + 1] + self.send_response(206) + self.send_header("Content-Range", f"bytes {start}-{end}/{len(BODY)}") + else: + # This is the defective-gateway behavior under test: ignore Range. + response_body = BODY + self.send_response(200) + + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + def log_message(self, format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), RangeHandler) + server_thread = Thread(target=server.serve_forever, daemon=True) + server_thread.start() + gateway_url = f"http://127.0.0.1:{server.server_address[1]}" + + try: + yield gateway_url, recorded_ranges + finally: + server.shutdown() + server.server_close() + server_thread.join() + + +def make_kubo_cas(gateway_url: str) -> KuboCAS: + return KuboCAS( + gateway_base_url=gateway_url, + rpc_base_url=gateway_url, + max_retries=0, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("cid", "load_kwargs", "expected"), + [ + ("ignores-bounded", {"offset": 5, "length": 10}, BODY[5:15]), + ("ignores-open-ended", {"offset": 5}, BODY[5:]), + ("ignores-suffix", {"suffix": 7}, BODY[-7:]), + ], + ids=["bounded", "open-ended", "suffix"], +) +async def test_kubocas_slices_full_body_when_gateway_ignores_range( + range_gateway: tuple[str, RecordedRanges], + cid: str, + load_kwargs: dict[str, int], + expected: bytes, +) -> None: + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + actual = await cas.load(cid, **load_kwargs) + finally: + await cas.aclose() + + assert actual == expected, ( + f"gateway returned {len(actual)} bytes; expected {len(expected)} " + f"for {load_kwargs}" + ) + + +@pytest.mark.asyncio +async def test_kubocas_zero_length_returns_empty_without_request( + range_gateway: tuple[str, RecordedRanges], +) -> None: + gateway_url, recorded_ranges = range_gateway + cid = "zero-length" + cas = make_kubo_cas(gateway_url) + try: + actual = await cas.load(cid, offset=5, length=0) + finally: + await cas.aclose() + + observed_ranges = recorded_ranges.get(cid, []) + assert (actual, observed_ranges) == (b"", []), ( + f"zero-length load returned {len(actual)} bytes and sent Range headers " + f"{observed_ranges!r}; expected 0 bytes and no request" + ) + + +@pytest.mark.asyncio +async def test_inmemorycas_zero_length_returns_empty() -> None: + cas = InMemoryCAS() + key = await cas.save(BODY, "raw") + + assert await cas.load(key, offset=5, length=0) == b"" + + +@pytest.mark.asyncio +async def test_inmemorycas_zero_suffix_returns_empty() -> None: + cas = InMemoryCAS() + key = await cas.save(BODY, "raw") + + actual = await cas.load(key, suffix=0) + + assert actual == b"", ( + f"suffix=0 returned the full {len(actual)}-byte object; expected b''" + ) + + +@pytest.mark.asyncio +async def test_kubocas_zero_suffix_returns_empty_without_request( + range_gateway: tuple[str, RecordedRanges], +) -> None: + gateway_url, recorded_ranges = range_gateway + cid = "zero-suffix" + cas = make_kubo_cas(gateway_url) + try: + actual = await cas.load(cid, suffix=0) + finally: + await cas.aclose() + + observed_ranges = recorded_ranges.get(cid, []) + assert (actual, observed_ranges) == (b"", []), ( + f"suffix=0 load returned {len(actual)} bytes and sent Range headers " + f"{observed_ranges!r}; expected 0 bytes and no request" + ) + + +@pytest.mark.asyncio +async def test_kubocas_proper_206_range_is_unchanged( + range_gateway: tuple[str, RecordedRanges], +) -> None: + gateway_url, recorded_ranges = range_gateway + cas = make_kubo_cas(gateway_url) + try: + actual = await cas.load("honors-range", offset=5, length=10) + finally: + await cas.aclose() + + assert actual == BODY[5:15] + assert recorded_ranges["honors-range"] == ["bytes=5-14"] + + +@pytest.mark.asyncio +async def test_inmemorycas_normal_ranges_use_python_slice_semantics() -> None: + cas = InMemoryCAS() + key = await cas.save(BODY, "raw") + + assert await cas.load(key, offset=5, length=10) == BODY[5:15] + assert await cas.load(key, offset=5) == BODY[5:] + assert await cas.load(key, suffix=7) == BODY[-7:] + + +@pytest.mark.asyncio +async def test_inmemorycas_offset_past_eof_returns_empty() -> None: + """Python-slice semantics: a read starting at/past EOF yields b"".""" + cas = InMemoryCAS() + key = await cas.save(BODY, "raw") + + assert await cas.load(key, offset=len(BODY)) == b"" + assert await cas.load(key, offset=len(BODY) + 10) == b"" + assert await cas.load(key, offset=len(BODY) + 10, length=5) == b"" + + +@pytest.mark.asyncio +async def test_kubocas_offset_past_eof_returns_empty( + range_gateway: tuple[str, RecordedRanges], +) -> None: + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + actual = await cas.load("range-not-satisfiable", offset=len(BODY) + 10) + finally: + await cas.aclose() + + assert actual == b"", ( + f"offset past EOF returned {len(actual)} bytes; expected b'' to match " + "Python-slice / InMemoryCAS semantics" + ) + + +@pytest.mark.asyncio +async def test_kubocas_in_bounds_416_is_surfaced_as_error( + range_gateway: tuple[str, RecordedRanges], +) -> None: + # A 416 whose ``*/N`` still covers the requested offset is a genuine + # (unexpected) error, not an empty read: it must not be swallowed. + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + with pytest.raises(httpx.HTTPStatusError): + await cas.load("range-not-satisfiable", offset=5) + finally: + await cas.aclose() + + +@pytest.mark.asyncio +async def test_kubocas_416_without_content_range_is_surfaced_as_error( + range_gateway: tuple[str, RecordedRanges], +) -> None: + # Without a parseable ``Content-Range`` we cannot prove the read is empty, + # so the 416 is raised rather than treated as b"". + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + with pytest.raises(httpx.HTTPStatusError): + await cas.load("malformed-416", offset=len(BODY) + 10) + finally: + await cas.aclose() + + +@pytest.mark.asyncio +async def test_kubocas_suffix_on_empty_object_returns_empty( + range_gateway: tuple[str, RecordedRanges], +) -> None: + # A suffix range against a zero-length object is unsatisfiable (416 bytes + # */0). Python slicing of b"" yields b"", so KuboCAS must too rather than + # raising -- mirroring InMemoryCAS and the offset-past-EOF case. + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + actual = await cas.load("empty-suffix-416", suffix=7) + finally: + await cas.aclose() + + assert actual == b"", ( + f"suffix read of an empty object returned {len(actual)} bytes; " + "expected b'' to match Python-slice / InMemoryCAS semantics" + ) + + +@pytest.mark.asyncio +async def test_inmemorycas_suffix_on_empty_object_returns_empty() -> None: + cas = InMemoryCAS() + key = await cas.save(b"", "raw") + + assert await cas.load(key, suffix=7) == b"" + + +@pytest.mark.asyncio +async def test_kubocas_honored_open_ended_and_suffix_206( + range_gateway: tuple[str, RecordedRanges], +) -> None: + # Valid 206 responses for an open-ended offset read and a suffix read must + # pass validation and return the exact requested window. + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + open_ended = await cas.load("honors-range", offset=5) + suffix = await cas.load("honors-suffix", suffix=7) + finally: + await cas.aclose() + + assert open_ended == BODY[5:] + assert suffix == BODY[-7:] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cid", + ["bad-206-no-range", "bad-206-star-total", "bad-206-bad-length"], +) +async def test_kubocas_rejects_inconsistent_206( + range_gateway: tuple[str, RecordedRanges], cid: str +) -> None: + # A 206 with a missing/unparseable Content-Range, or one whose declared + # window disagrees with the body length, is untrustworthy and must raise + # rather than silently corrupt the read. + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + with pytest.raises(httpx.HTTPStatusError): + await cas.load(cid, offset=5, length=10) + finally: + await cas.aclose() + + +@pytest.mark.asyncio +async def test_kubocas_rejects_206_with_wrong_window( + range_gateway: tuple[str, RecordedRanges], +) -> None: + # An internally-consistent 206 whose window does not start where we asked + # would hand back the wrong bytes; it must be rejected. + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + with pytest.raises(httpx.HTTPStatusError): + await cas.load("bad-206-wrong-start", offset=5, length=10) + finally: + await cas.aclose() + + +@pytest.mark.asyncio +async def test_kubocas_rejects_unexpected_success_status_for_range( + range_gateway: tuple[str, RecordedRanges], +) -> None: + # A non-200/206 success (here 203) to a Range request carries an unknown + # byte window and must be rejected rather than returned as-is. + gateway_url, _ = range_gateway + cas = make_kubo_cas(gateway_url) + try: + with pytest.raises(httpx.HTTPStatusError): + await cas.load("unexpected-2xx", offset=5, length=10) + finally: + await cas.aclose() diff --git a/tests/test_k5_follow_redirects.py b/tests/test_k5_follow_redirects.py new file mode 100644 index 0000000..1bf26ba --- /dev/null +++ b/tests/test_k5_follow_redirects.py @@ -0,0 +1,66 @@ +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from py_hamt import KuboCAS + +REDIRECT_CID = "redirect" +EXPECTED_BODY = b"loaded after following the gateway redirect" + + +class _RedirectGatewayHandler(BaseHTTPRequestHandler): + """Redirect one CID path to a successful local gateway response.""" + + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: + pass + + def do_GET(self) -> None: + if self.path == f"/ipfs/{REDIRECT_CID}": + self.send_response(301) + self.send_header("Location", "/ipfs/plain") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if self.path == "/ipfs/plain": + self.send_response(200) + self.send_header("Content-Length", str(len(EXPECTED_BODY))) + self.end_headers() + self.wfile.write(EXPECTED_BODY) + return + + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + +@pytest.fixture +def redirect_gateway_url() -> Iterator[str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), _RedirectGatewayHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + server_thread.join() + + +@pytest.mark.asyncio +async def test_internal_client_follows_gateway_redirect( + redirect_gateway_url: str, +) -> None: + cas = KuboCAS( + gateway_base_url=redirect_gateway_url, + rpc_base_url=redirect_gateway_url, + max_retries=0, + ) + try: + assert await cas.load(REDIRECT_CID) == EXPECTED_BODY + finally: + await cas.aclose() diff --git a/tests/test_k6_retry_policy.py b/tests/test_k6_retry_policy.py new file mode 100644 index 0000000..df60a27 --- /dev/null +++ b/tests/test_k6_retry_policy.py @@ -0,0 +1,304 @@ +import socket +import threading +from collections.abc import Iterator +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import httpx +import pytest +from multiformats import CID + +from py_hamt import KuboCAS, store_httpx + +EXPECTED_BODY = b"gateway response after transient failures" +TEST_CID = CID.decode("bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad7lrm") + + +@pytest.fixture +def retrying_kubo_server() -> Iterator[tuple[str, dict[str, int]]]: + """Serve deterministic transient and permanent Kubo HTTP responses.""" + hit_counts: dict[str, int] = {} + + class RetryHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, message_format: str, *args: object) -> None: + pass + + def _record_hit(self, key: str) -> int: + hit_counts[key] = hit_counts.get(key, 0) + 1 + return hit_counts[key] + + def _send_response( + self, + status: int, + body: bytes = b"", + headers: dict[str, str] | None = None, + ) -> None: + self.send_response(status) + for name, value in (headers or {}).items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + def do_GET(self) -> None: + cid = self.path.rsplit("/", 1)[-1].split("?", 1)[0] + attempt = self._record_hit(cid) + + if cid == "transient-500" and attempt <= 2: + self._send_response(500) + return + if cid == "rate-limited" and attempt == 1: + self._send_response(429, headers={"Retry-After": "0"}) + return + if cid == "missing": + self._send_response(404) + return + self._send_response(200, EXPECTED_BODY) + + def do_POST(self) -> None: + content_length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(content_length) + attempt = self._record_hit("POST") + if attempt <= 2: + self._send_response(500) + return + + response_body = ('{"Hash":"' + str(TEST_CID) + '"}').encode() + self._send_response( + 200, + response_body, + headers={"Content-Type": "application/json"}, + ) + + server = ThreadingHTTPServer(("127.0.0.1", 0), RetryHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + url = f"http://127.0.0.1:{server.server_address[1]}" + + try: + yield url, hit_counts + finally: + server.shutdown() + server.server_close() + server_thread.join() + + +def make_cas(url: str, *, max_retries: int = 3) -> KuboCAS: + return KuboCAS( + gateway_base_url=url, + rpc_base_url=url, + max_retries=max_retries, + initial_delay=0.01, + ) + + +def test_retry_delay_parses_http_dates_and_ignores_invalid_values( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # parsedate_to_datetime yields a naive datetime when the header lacks a + # timezone; _retry_delay treats such values as UTC, so anchor the fake + # future time to UTC (not local) to stay timezone-independent. + future_naive_datetime = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta( + seconds=30 + ) + monkeypatch.setattr( + store_httpx, + "parsedate_to_datetime", + lambda _: future_naive_datetime, + ) + dated_response = httpx.Response(429, headers={"Retry-After": "future-date"}) + delay = store_httpx._retry_delay(10, 2, 1, dated_response) + assert 28 <= delay <= 30 + + def reject_retry_after(_: str) -> datetime: + raise ValueError("invalid Retry-After") + + monkeypatch.setattr(store_httpx, "parsedate_to_datetime", reject_retry_after) + monkeypatch.setattr(store_httpx.random, "random", lambda: 0.5) + invalid_response = httpx.Response(429, headers={"Retry-After": "invalid"}) + assert store_httpx._retry_delay(10, 2, 1, invalid_response) == 10 + + +def test_retry_delay_numeric_retry_after_not_shortened_by_backoff() -> None: + response = httpx.Response(429, headers={"Retry-After": "7"}) + + delay = store_httpx._retry_delay(1, 2, 1, response) + + assert delay == 7 + + +def test_retry_delay_http_date_retry_after_not_shortened_by_backoff() -> None: + retry_at = datetime.now(timezone.utc) + timedelta(seconds=7) + response = httpx.Response( + 503, + headers={"Retry-After": format_datetime(retry_at, usegmt=True)}, + ) + + delay = store_httpx._retry_delay(1, 2, 1, response) + + assert 5 <= delay <= 7 + + +@pytest.mark.parametrize("header", ["inf", "1e309", "999999"]) +def test_retry_delay_caps_pathological_retry_after(header: str) -> None: + response = httpx.Response(429, headers={"Retry-After": header}) + + delay = store_httpx._retry_delay(1, 2, 1, response) + + assert delay == store_httpx._MAX_RETRY_AFTER_SECONDS + + +def test_retry_delay_ignores_nan_retry_after() -> None: + response = httpx.Response(429, headers={"Retry-After": "nan"}) + + delay = store_httpx._retry_delay(10, 2, 1, response) + + assert 9 <= delay <= 11 + + +def test_retry_delay_preserves_short_retry_after() -> None: + response = httpx.Response(429, headers={"Retry-After": "1.5"}) + + delay = store_httpx._retry_delay(2, 2, 2, response) + + assert delay == 1.5 + + +def test_slice_requested_range_handles_zero_suffix() -> None: + assert store_httpx._slice_requested_range(b"content", None, None, 0) == b"" + + +def test_kubo_cas_rejects_negative_concurrency() -> None: + with pytest.raises(ValueError, match="concurrency must be a positive integer"): + KuboCAS(concurrency=-1) + + +def test_kubo_cas_rejects_zero_concurrency() -> None: + with pytest.raises(ValueError, match="concurrency must be a positive integer"): + KuboCAS(concurrency=0) + + +@pytest.mark.asyncio +async def test_owned_kubo_cas_reopens_its_semaphore_after_close() -> None: + cas = KuboCAS() + cas._closed = True + + semaphore = cas._loop_semaphore() + + assert isinstance(semaphore, store_httpx.asyncio.Semaphore) + assert cas._closed is False + await cas.aclose() + + +@pytest.mark.asyncio +async def test_closed_kubo_cas_with_supplied_client_cannot_reopen() -> None: + client = httpx.AsyncClient() + cas = KuboCAS(client=client) + await cas.aclose() + try: + with pytest.raises(RuntimeError, match="KuboCAS is closed"): + cas._loop_semaphore() + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_save_does_not_retry_nonretryable_status() -> None: + request_count = 0 + + async def reject_save(request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response(400, request=request) + + client = httpx.AsyncClient(transport=httpx.MockTransport(reject_save)) + cas = KuboCAS(client=client, max_retries=3) + try: + with pytest.raises(httpx.HTTPStatusError) as error: + await cas.save(b"invalid upload", codec="raw") + finally: + await cas.aclose() + await client.aclose() + + assert error.value.response.status_code == 400 + assert request_count == 1 + + +@pytest.mark.asyncio +async def test_load_retries_transient_500_responses( + retrying_kubo_server: tuple[str, dict[str, int]], +) -> None: + url, hit_counts = retrying_kubo_server + cas = make_cas(url) + try: + result = await cas.load("transient-500") + finally: + await cas.aclose() + + assert result == EXPECTED_BODY + assert hit_counts["transient-500"] == 3 + + +@pytest.mark.asyncio +async def test_load_retries_429_with_retry_after( + retrying_kubo_server: tuple[str, dict[str, int]], +) -> None: + url, hit_counts = retrying_kubo_server + cas = make_cas(url) + try: + result = await cas.load("rate-limited") + finally: + await cas.aclose() + + assert result == EXPECTED_BODY + assert hit_counts["rate-limited"] == 2 + + +@pytest.mark.asyncio +async def test_load_preserves_connect_error_after_retries() -> None: + with socket.socket() as ephemeral_socket: + ephemeral_socket.bind(("127.0.0.1", 0)) + dead_port = ephemeral_socket.getsockname()[1] + + dead_url = f"http://127.0.0.1:{dead_port}" + cas = make_cas(dead_url, max_retries=2) + try: + with pytest.raises(httpx.ConnectError): + await cas.load("unreachable") + finally: + await cas.aclose() + + +@pytest.mark.asyncio +async def test_load_does_not_retry_404( + retrying_kubo_server: tuple[str, dict[str, int]], +) -> None: + url, hit_counts = retrying_kubo_server + cas = make_cas(url) + try: + with pytest.raises(httpx.HTTPStatusError) as error: + await cas.load("missing") + finally: + await cas.aclose() + + assert error.value.response.status_code == 404 + assert hit_counts["missing"] == 1 + + +@pytest.mark.asyncio +async def test_save_retries_transient_500_responses( + retrying_kubo_server: tuple[str, dict[str, int]], +) -> None: + url, hit_counts = retrying_kubo_server + cas = make_cas(url) + try: + result = await cas.save(b"content-addressed upload", codec="raw") + finally: + await cas.aclose() + + assert isinstance(result, CID) + assert hit_counts["POST"] == 3 diff --git a/tests/test_k7_url_normalization.py b/tests/test_k7_url_normalization.py new file mode 100644 index 0000000..8d687d9 --- /dev/null +++ b/tests/test_k7_url_normalization.py @@ -0,0 +1,27 @@ +import pytest + +from py_hamt.store_httpx import KuboCAS + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "gateway_base_url", + [ + "https://example.com", + "https://example.com/", + "https://example.com/ipfs", + "https://example.com/ipfs/", + ], +) +async def test_gateway_base_url_normalizes_ipfs_path( + gateway_base_url: str, +) -> None: + cas = KuboCAS( + gateway_base_url=gateway_base_url, + rpc_base_url="https://example.com", + ) + + try: + assert cas.gateway_base_url == "https://example.com/ipfs/" + finally: + await cas.aclose() diff --git a/tests/test_k8_respect_client_timeout.py b/tests/test_k8_respect_client_timeout.py new file mode 100644 index 0000000..af39ed3 --- /dev/null +++ b/tests/test_k8_respect_client_timeout.py @@ -0,0 +1,82 @@ +import threading +import time +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import httpx +import pytest +from multiformats import CID + +from py_hamt import KuboCAS + +SLOW_RESPONSE_DELAY = 0.5 +USER_TIMEOUT = 0.05 +TEST_CID = CID.decode("bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad7lrm") + + +class _SlowKuboHandler(BaseHTTPRequestHandler): + """Serve slow gateway and RPC responses without external network access.""" + + def log_message(self, format: str, *args: object) -> None: + pass + + def _send_slow_response(self, body: bytes, content_type: str) -> None: + time.sleep(SLOW_RESPONSE_DELAY) + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + # Once the defect is fixed, the client closes the timed-out socket. + pass + + def do_GET(self) -> None: + self._send_slow_response(b"slow gateway response", "application/octet-stream") + + def do_POST(self) -> None: + body = ('{"Hash":"' + str(TEST_CID) + '"}').encode() + self._send_slow_response(body, "application/json") + + +@pytest.fixture +def slow_kubo_url() -> Iterator[str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), _SlowKuboHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + port = server.server_address[1] + try: + yield f"http://127.0.0.1:{port}" + finally: + server.shutdown() + server.server_close() + server_thread.join() + + +@pytest.mark.asyncio +async def test_load_respects_user_client_timeout(slow_kubo_url: str) -> None: + async with httpx.AsyncClient(timeout=USER_TIMEOUT) as client: + cas = KuboCAS( + client=client, + gateway_base_url=slow_kubo_url, + rpc_base_url=slow_kubo_url, + max_retries=0, + ) + + with pytest.raises(httpx.TimeoutException): + await cas.load(TEST_CID) + + +@pytest.mark.asyncio +async def test_save_respects_user_client_timeout(slow_kubo_url: str) -> None: + async with httpx.AsyncClient(timeout=USER_TIMEOUT) as client: + cas = KuboCAS( + client=client, + gateway_base_url=slow_kubo_url, + rpc_base_url=slow_kubo_url, + max_retries=0, + ) + + with pytest.raises(httpx.TimeoutException): + await cas.save(b"slow RPC request", codec="raw") diff --git a/tests/test_k9_semaphore_scope.py b/tests/test_k9_semaphore_scope.py new file mode 100644 index 0000000..21982b1 --- /dev/null +++ b/tests/test_k9_semaphore_scope.py @@ -0,0 +1,56 @@ +import asyncio +import time + +import httpx +import pytest + +from py_hamt import KuboCAS + + +@pytest.mark.asyncio +async def test_load_releases_semaphore_during_retry_backoff() -> None: + """A failing CID's retry delays must not consume the only request slot.""" + expected_body = b"healthy gateway response" + + def handle_request(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/bad-cid"): + raise httpx.ConnectError("simulated connection failure", request=request) + return httpx.Response(200, content=expected_body, request=request) + + async with httpx.AsyncClient( + transport=httpx.MockTransport(handle_request) + ) as client: + cas = KuboCAS( + client=client, + gateway_base_url="https://gateway.test", + rpc_base_url="https://rpc.test", + concurrency=1, + max_retries=2, + initial_delay=0.4, + backoff_factor=2.0, + ) + + async def failing_load() -> bytes: + return await cas.load("bad-cid") + + async def healthy_load() -> tuple[bytes, float]: + await asyncio.sleep(0.05) + started_at = time.perf_counter() + body = await cas.load("good-cid") + return body, time.perf_counter() - started_at + + try: + failure, healthy_result = await asyncio.gather( + failing_load(), healthy_load(), return_exceptions=True + ) + finally: + await cas.aclose() + + assert isinstance(failure, httpx.ConnectError) + assert not isinstance(healthy_result, BaseException) + healthy_body, healthy_elapsed = healthy_result + assert healthy_body == expected_body + assert healthy_elapsed < 0.5, ( + "healthy load waited behind another load's retry backoff: " + f"{healthy_elapsed:.3f}s elapsed" + ) diff --git a/tests/test_kubo_cas.py b/tests/test_kubo_cas.py index a508313..7d056af 100644 --- a/tests/test_kubo_cas.py +++ b/tests/test_kubo_cas.py @@ -6,7 +6,7 @@ import httpx import pytest from dag_cbor import IPLDKind -from hypothesis import given, settings +from hypothesis import HealthCheck, given, settings from testing_utils import ipld_strategy # noqa from py_hamt import InMemoryCAS, KuboCAS @@ -46,9 +46,13 @@ async def test_memory_store_invalid_key_type(): # Test that always works with Docker or local daemon @pytest.mark.ipfs -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio @given(data=ipld_strategy()) -@settings(deadline=1000, print_blob=True) +@settings( + deadline=1000, + print_blob=True, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) async def test_kubo_urls_explicit(create_ipfs, global_client_session, data: IPLDKind): """ Tests KuboCAS functionality with explicitly provided URLs. @@ -72,9 +76,13 @@ async def test_kubo_urls_explicit(create_ipfs, global_client_session, data: IPLD @pytest.mark.ipfs -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio @given(data=ipld_strategy()) -@settings(deadline=1000, print_blob=True) +@settings( + deadline=1000, + print_blob=True, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) async def test_kubo_default_urls(global_client_session, data: IPLDKind): """ Tests KuboCAS using its default URLs and when None is passed for URLs. @@ -248,13 +256,13 @@ async def failing_method(url, **kwargs): ): with pytest.raises( httpx.TimeoutException, - match="Failed to save data after 3 retries", + match="Simulated timeout", ): await kubo_cas.save(test_data, codec="dag-cbor") with pytest.raises( httpx.TimeoutException, - match="Failed to load data after 3 retries", + match="Simulated timeout", ): await kubo_cas.load(cid) @@ -305,9 +313,9 @@ async def mock_sleep(delay): @pytest.mark.asyncio -async def test_kubo_http_status_error_no_retry(): +async def test_kubo_http_status_error_retries_transient_status(): """ - Tests that KuboCAS immediately raises HTTPStatusError without retrying. + Tests that KuboCAS retries HTTP 500 before raising HTTPStatusError. """ # This mock simulates a server error by returning a 500 status code. @@ -321,7 +329,7 @@ async def mock_post_server_error(url, **kwargs): with patch.object( httpx.AsyncClient, "post", new=AsyncMock(side_effect=mock_post_server_error) ): - # Also patch asyncio.sleep to verify it's not called (i.e., no retries). + # Patch asyncio.sleep so retry backoff does not slow the test. with patch("asyncio.sleep", new=AsyncMock()) as mock_sleep: async with httpx.AsyncClient() as client: async with KuboCAS(client=client) as kubo_cas: @@ -331,8 +339,8 @@ async def mock_post_server_error(url, **kwargs): # Verify that the response in the exception has the correct status code. assert exc_info.value.response.status_code == 500 - # Verify that no retry was attempted. - mock_sleep.assert_not_called() + # The initial attempt is followed by three retries. + assert mock_sleep.await_count == kubo_cas.max_retries @pytest.mark.asyncio diff --git a/tests/test_kubo_pin.py b/tests/test_kubo_pin.py index 703dc92..10edef4 100644 --- a/tests/test_kubo_pin.py +++ b/tests/test_kubo_pin.py @@ -4,7 +4,7 @@ from py_hamt import KuboCAS -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio async def test_pinning(create_ipfs, global_client_session): """ Tests pinning a CID using KuboCAS with explicit URLs. diff --git a/tests/test_p1_direct_blake3.py b/tests/test_p1_direct_blake3.py new file mode 100644 index 0000000..8f894e6 --- /dev/null +++ b/tests/test_p1_direct_blake3.py @@ -0,0 +1,66 @@ +from typing import Any + +import pytest +from blake3 import blake3 +from multiformats import multihash + +from py_hamt.hamt import blake3_hashfn + + +@pytest.mark.parametrize( + ("input_bytes", "expected_hex_digest"), + [ + (b"", "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"), + (b"\x00", "2d3adedff11b61f14c886e35afa036736dcd87a74d27b5c1510225d0f592e213"), + ( + b"hello world", + "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24", + ), + ( + "héllo ünïcöde".encode(), + "90ede513eb21f2db0fc88dc008be5e07d35304d4628624c614abc5a25991d5f3", + ), + ( + bytes(range(256)) * 4096, + "64479cf7293960210547db8d982359e0c4ce054525ed7086cf93030828fc0533", + ), + ], + ids=["empty", "one-byte", "ascii", "unicode", "one-megabyte-pattern"], +) +def test_blake3_hashfn_preserves_golden_digest( + input_bytes: bytes, expected_hex_digest: str +) -> None: + digest = blake3_hashfn(input_bytes) + + assert len(digest) == 32 + assert digest.hex() == expected_hex_digest + + +def test_blake3_hashfn_bypasses_multiformats_wrappers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + data = b"structural probe: no multiformats in hot path" + multiformats_call_count = 0 + original_digest = multihash.Multihash.digest + original_unwrap = multihash.Multihash.unwrap + + def digest_spy(self: multihash.Multihash, *args: Any, **kwargs: Any) -> bytes: + nonlocal multiformats_call_count + multiformats_call_count += 1 + return original_digest(self, *args, **kwargs) + + def unwrap_spy(self: multihash.Multihash, *args: Any, **kwargs: Any) -> bytes: + nonlocal multiformats_call_count + multiformats_call_count += 1 + return original_unwrap(self, *args, **kwargs) + + monkeypatch.setattr(multihash.Multihash, "digest", digest_spy) + monkeypatch.setattr(multihash.Multihash, "unwrap", unwrap_spy) + + digest = blake3_hashfn(data) + + assert digest == blake3(data).digest(length=32) + assert multiformats_call_count == 0, ( + "blake3_hashfn must not route through multiformats multihash wrappers " + "(perf: ~134us/hash overhead)" + ) diff --git a/tests/test_p2_relink_o1_hash_once.py b/tests/test_p2_relink_o1_hash_once.py new file mode 100644 index 0000000..dc1b528 --- /dev/null +++ b/tests/test_p2_relink_o1_hash_once.py @@ -0,0 +1,46 @@ +import pytest +from dag_cbor import IPLDKind + +from py_hamt import InMemoryCAS +from py_hamt.hamt import HAMT, Node + + +@pytest.mark.asyncio +async def test_golden_root_id_500_keys() -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + + for index in range(500): + await hamt.set(f"key-{index}", f"value-{index}") + + await hamt.make_read_only() + + assert bytes(hamt.root_node_id).hex() == ( + "1e20330cb3ccd5ef3940490f30afd6a852e1e7545df16e9ed2b754405c883ba0cab0" + ) + + +@pytest.mark.asyncio +async def test_get_link_calls_bounded_during_bulk_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + number_of_keys = 3_000 + get_link_calls = 0 + original_get_link = Node.get_link + + def counting_get_link(node: Node, index: int) -> IPLDKind: + nonlocal get_link_calls + get_link_calls += 1 + return original_get_link(node, index) + + monkeypatch.setattr(Node, "get_link", counting_get_link) + + hamt = await HAMT.build(cas=InMemoryCAS()) + for index in range(number_of_keys): + await hamt.set(f"key-{index}", f"value-{index}") + + maximum_get_link_calls = 20 * number_of_keys + # Pre-fix measurement: 194,838 calls for 3,000 keys (bound: 60,000). + assert get_link_calls <= maximum_get_link_calls, ( + f"get_link called {get_link_calls} times for {number_of_keys} sets; " + f"expected at most {maximum_get_link_calls} (20 * N)" + ) diff --git a/tests/test_p4_cid_objects.py b/tests/test_p4_cid_objects.py new file mode 100644 index 0000000..5b49bd8 --- /dev/null +++ b/tests/test_p4_cid_objects.py @@ -0,0 +1,107 @@ +import pytest +import zarr.core.buffer +from multiformats import CID +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +METADATA = b'{"shape":[2,2],"node_type":"array"}' + + +async def _new_store(cas: CIDInMemoryCAS) -> ShardedZarrStore: + return await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(2, 2), + chunk_shape=(1, 1), + chunks_per_shard=2, + ) + + +@pytest.mark.asyncio +async def test_warm_gets_and_sets_keep_cids_as_objects( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cas = CIDInMemoryCAS() + store = await _new_store(cas) + initial_values = { + "zarr.json": METADATA, + "c/0/0": b"chunk-00", + "c/0/1": b"chunk-01", + "c/1/0": b"chunk-10", + } + for key, value in initial_values.items(): + await store.set(key, PROTOTYPE.buffer.from_bytes(value)) + + warm_keys = ("c/0/0", "c/0/1", "c/1/0") + for key in warm_keys: + assert await store.get(key, PROTOTYPE) is not None + + counters = {"str": 0, "decode": 0} + original_decode = CID.decode + original_str = CID.__str__ + + def counted_decode(value: str | bytes) -> CID: + counters["decode"] += 1 + return original_decode(value) + + def counted_str(cid: CID) -> str: + counters["str"] += 1 + return original_str(cid) + + monkeypatch.setattr(CID, "decode", staticmethod(counted_decode)) + monkeypatch.setattr(CID, "__str__", counted_str) + + counters.update(str=0, decode=0) + for key in warm_keys: + result = await store.get(key, PROTOTYPE) + assert result is not None + assert result.to_bytes() == initial_values[key] + + warm_get_counters = counters.copy() + counters.update(str=0, decode=0) + for key, value in { + "c/1/0": b"updated-chunk-10", + "c/1/1": b"chunk-11", + }.items(): + await store.set(key, PROTOTYPE.buffer.from_bytes(value)) + set_counters = counters.copy() + + assert warm_get_counters == {"str": 0, "decode": 0}, ( + "warm gets must pass CID objects directly; observed " + f"{warm_get_counters['str']} CID.__str__ calls and " + f"{warm_get_counters['decode']} CID.decode calls" + ) + assert set_counters["decode"] == 0, ( + "sets must keep saved CIDs as objects instead of decoding their strings; " + f"observed {set_counters['decode']} CID.decode calls" + ) + + +@pytest.mark.asyncio +async def test_cid_object_optimization_preserves_root_and_values() -> None: + cas = CIDInMemoryCAS() + store = await _new_store(cas) + values = { + "zarr.json": METADATA, + "c/0/0": b"golden-00", + "c/0/1": b"golden-01", + "c/1/0": b"golden-10", + "c/1/1": b"golden-11", + } + for key, value in values.items(): + await store.set(key, PROTOTYPE.buffer.from_bytes(value)) + + root_cid = await store.flush() + + assert root_cid == "bafyr4iewmdsuckgrb44tf7xie6jj5tuaany3he33r5qh352e37olgvuzhy" + read_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + for key, expected in values.items(): + result = await read_store.get(key, PROTOTYPE) + assert result is not None + assert result.to_bytes() == expected diff --git a/tests/test_p5_concurrent_vacate.py b/tests/test_p5_concurrent_vacate.py new file mode 100644 index 0000000..a94a87b --- /dev/null +++ b/tests/test_p5_concurrent_vacate.py @@ -0,0 +1,98 @@ +"""Regression coverage for concurrent HAMT cache vacating. + +Vacating must flush independent sibling subtrees concurrently without violating +the required children-before-parents ordering; the root must still be saved last. +""" + +import asyncio +from typing import NamedTuple + +import pytest +import pytest_asyncio + +from py_hamt import InMemoryCAS +from py_hamt.hamt import HAMT +from py_hamt.store_httpx import ContentAddressedStore + +NUMBER_OF_KEYS = 300 +GOLDEN_ROOT_HEX = "1e205c7ba99421c5013ad193acef6f7ee70140051bdbf32d4018d7351b8e3b9f3765" + + +class DelayedInMemoryCAS(InMemoryCAS): + """An InMemoryCAS that exposes concurrent save calls without changing IDs.""" + + def __init__(self) -> None: + super().__init__() + self.in_flight = 0 + self.max_in_flight = 0 + + def reset_concurrency_counters(self) -> None: + self.in_flight = 0 + self.max_in_flight = 0 + + async def save( + self, + data: bytes, + codec: ContentAddressedStore.CodecInput, + ) -> bytes: + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + try: + await asyncio.sleep(0.01) + return await super().save(data, codec) + finally: + self.in_flight -= 1 + + +class VacateResult(NamedTuple): + cas: DelayedInMemoryCAS + root_node_id: bytes + max_in_flight: int + + +def key_value(index: int) -> tuple[str, bytes]: + return f"key-{index}", f"value-{index}".encode() + + +@pytest_asyncio.fixture(scope="module", loop_scope="module") +async def vacate_result() -> VacateResult: + cas = DelayedInMemoryCAS() + hamt = await HAMT.build( + cas=cas, + max_bucket_size=1, + values_are_bytes=True, + ) + for index in range(NUMBER_OF_KEYS): + key, value = key_value(index) + await hamt.set(key, value) + + cas.reset_concurrency_counters() + await hamt.cache_vacate() + + return VacateResult( + cas=cas, + root_node_id=bytes(hamt.root_node_id), + max_in_flight=cas.max_in_flight, + ) + + +@pytest.mark.asyncio +async def test_vacate_golden_root_and_readable(vacate_result: VacateResult) -> None: + assert vacate_result.root_node_id.hex() == GOLDEN_ROOT_HEX + + read_hamt = await HAMT.build( + cas=vacate_result.cas, + root_node_id=vacate_result.root_node_id, + read_only=True, + values_are_bytes=True, + ) + for index in range(NUMBER_OF_KEYS): + key, value = key_value(index) + assert await read_hamt.get(key) == value + + +def test_vacate_flushes_siblings_concurrently(vacate_result: VacateResult) -> None: + assert vacate_result.max_in_flight > 1, ( + "vacate() should overlap CAS saves for independent sibling subtrees; " + f"observed max in-flight saves: {vacate_result.max_in_flight}" + ) diff --git a/tests/test_p6_bounded_vacate.py b/tests/test_p6_bounded_vacate.py new file mode 100644 index 0000000..7b8a4e2 --- /dev/null +++ b/tests/test_p6_bounded_vacate.py @@ -0,0 +1,94 @@ +import asyncio +from typing import cast + +import pytest + +from py_hamt import HAMT, ContentAddressedStore, InMemoryCAS +from py_hamt.hamt import InMemoryTreeStore + + +class InstrumentedCAS(InMemoryCAS): + def __init__(self) -> None: + super().__init__() + self.in_flight_saves = 0 + self.max_concurrent_saves = 0 + + async def save(self, data: bytes, codec: ContentAddressedStore.CodecInput) -> bytes: + self.in_flight_saves += 1 + self.max_concurrent_saves = max(self.max_concurrent_saves, self.in_flight_saves) + try: + await asyncio.sleep(0.002) + return await super().save(data, codec) + finally: + self.in_flight_saves -= 1 + + def reset_save_concurrency(self) -> None: + self.in_flight_saves = 0 + self.max_concurrent_saves = 0 + + +class FailingCAS(InstrumentedCAS): + def __init__(self, fail_on_save_number: int) -> None: + super().__init__() + self.fail_on_save_number = fail_on_save_number + self.armed = False + self.armed_save_calls = 0 + + def arm(self) -> None: + self.armed = True + self.armed_save_calls = 0 + + async def save(self, data: bytes, codec: ContentAddressedStore.CodecInput) -> bytes: + if self.armed: + self.armed_save_calls += 1 + if self.armed_save_calls == self.fail_on_save_number: + # Let the rest of the unbounded wave start before this task fails. + await asyncio.sleep(0) + raise ConnectionError("simulated CAS save failure") + + self.in_flight_saves += 1 + self.max_concurrent_saves = max(self.max_concurrent_saves, self.in_flight_saves) + try: + await asyncio.sleep(0.01) + return await InMemoryCAS.save(self, data, codec) + finally: + self.in_flight_saves -= 1 + + +async def build_wide_hamt(cas: InMemoryCAS) -> HAMT: + hamt = await HAMT.build(cas=cas) + for i in range(2000): + await hamt.set(str(i), b"v") + return hamt + + +@pytest.mark.asyncio +async def test_vacate_save_waves_are_bounded() -> None: + cas = InstrumentedCAS() + hamt = await build_wide_hamt(cas) + cas.reset_save_concurrency() + + async with hamt.lock: + await cast(InMemoryTreeStore, hamt.node_store).vacate() + + assert cas.max_concurrent_saves <= 16, "vacate save waves must be bounded" + + +@pytest.mark.asyncio +async def test_failed_vacate_leaves_no_orphan_save_tasks() -> None: + cas = FailingCAS(fail_on_save_number=5) + hamt = await build_wide_hamt(cas) + cas.arm() + + baseline_tasks = asyncio.all_tasks() + with pytest.raises(ConnectionError): + await hamt.cache_vacate() + + pending = [ + task + for task in asyncio.all_tasks() + if task not in baseline_tasks + and task is not asyncio.current_task() + and not task.done() + ] + assert pending == [], f"orphaned vacate save tasks survived the failure: {pending}" diff --git a/tests/test_p7_len_without_snapshot.py b/tests/test_p7_len_without_snapshot.py new file mode 100644 index 0000000..319c709 --- /dev/null +++ b/tests/test_p7_len_without_snapshot.py @@ -0,0 +1,40 @@ +from collections.abc import AsyncIterator + +import pytest + +from py_hamt import HAMT, InMemoryCAS + + +@pytest.mark.asyncio +async def test_len_does_not_materialize_key_snapshot_via_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + hamt = await HAMT.build(cas=InMemoryCAS()) + for index in range(50): + await hamt.set(f"key-{index}", f"value-{index}") + + async def keys_must_not_be_called() -> AsyncIterator[str]: + raise AssertionError("len() must not materialize a key snapshot via keys()") + yield + + monkeypatch.setattr(hamt, "keys", keys_must_not_be_called) + + assert await hamt.len() == 50 + + +@pytest.mark.asyncio +async def test_len_matches_set_and_delete_operations_in_write_and_read_only_modes() -> ( + None +): + hamt = await HAMT.build(cas=InMemoryCAS()) + + for index in range(12): + await hamt.set(f"key-{index}", f"value-{index}") + assert await hamt.len() == 12 + + await hamt.delete("key-2") + await hamt.delete("key-9") + assert await hamt.len() == 10 + + await hamt.make_read_only() + assert await hamt.len() == 10 diff --git a/tests/test_p8_bounded_task_creation.py b/tests/test_p8_bounded_task_creation.py new file mode 100644 index 0000000..a96a8d6 --- /dev/null +++ b/tests/test_p8_bounded_task_creation.py @@ -0,0 +1,257 @@ +import asyncio +import contextvars +import json +from collections.abc import Coroutine +from typing import Any, TypeVar, cast + +import pytest +import zarr +from multiformats import CID +from testing_utils import CIDInMemoryCAS + +from py_hamt import HAMT, ContentAddressedStore, InMemoryCAS, ShardedZarrStore +from py_hamt.hamt import _VACATE_CONCURRENCY, InMemoryTreeStore +from py_hamt.sharded_zarr_store import _FLUSH_CONCURRENCY + +_T = TypeVar("_T") +_HAMT_KEY_COUNT = 300 +_SHARD_COUNT = 40 +_SAVE_DELAY_SECONDS = 0.01 + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [_SHARD_COUNT * 2, 2], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2, 2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes"}], + }, + separators=(",", ":"), +).encode() + + +class TaskCreationTracker: + """Count tasks created during one flush until each task finishes.""" + + def __init__(self) -> None: + self.live_tasks: set[asyncio.Task[Any]] = set() + self.peak_live_tasks = 0 + + def install(self, monkeypatch: pytest.MonkeyPatch) -> None: + loop = asyncio.get_running_loop() + original_create_task = loop.create_task + + def tracked_create_task( + coro: Coroutine[Any, Any, _T], + *, + name: str | None = None, + context: contextvars.Context | None = None, + ) -> asyncio.Task[_T]: + task = original_create_task(coro, name=name, context=context) + untyped_task = cast(asyncio.Task[Any], task) + self.live_tasks.add(untyped_task) + self.peak_live_tasks = max( + self.peak_live_tasks, + len(self.live_tasks), + ) + task.add_done_callback(self.live_tasks.discard) + return task + + monkeypatch.setattr(loop, "create_task", tracked_create_task) + + def pending_tasks(self) -> list[asyncio.Task[Any]]: + return [task for task in self.live_tasks if not task.done()] + + +class DelayedBytesCAS(InMemoryCAS): + def __init__(self, fail_on_save: int | None = None) -> None: + super().__init__() + self.fail_on_save = fail_on_save + self.armed = False + self.started_saves = 0 + self.finished_saves = 0 + self.in_flight_saves = 0 + + def arm(self) -> None: + self.armed = True + self.started_saves = 0 + self.finished_saves = 0 + self.in_flight_saves = 0 + + async def save( + self, + data: bytes, + codec: ContentAddressedStore.CodecInput, + ) -> bytes: + if not self.armed: + return await super().save(data, codec) + + self.started_saves += 1 + save_number = self.started_saves + self.in_flight_saves += 1 + try: + if save_number == self.fail_on_save: + await asyncio.sleep(0) + raise ConnectionError("simulated HAMT save failure") + await asyncio.sleep(_SAVE_DELAY_SECONDS) + result = await super().save(data, codec) + self.finished_saves += 1 + return result + finally: + self.in_flight_saves -= 1 + + +class DelayedCIDCAS(CIDInMemoryCAS): + def __init__(self, fail_on_save: int | None = None) -> None: + super().__init__() + self.fail_on_save = fail_on_save + self.armed = False + self.started_saves = 0 + self.finished_saves = 0 + self.in_flight_saves = 0 + + def arm(self) -> None: + self.armed = True + self.started_saves = 0 + self.finished_saves = 0 + self.in_flight_saves = 0 + + async def save( + self, + data: bytes, + codec: ContentAddressedStore.CodecInput, + ) -> CID: + if not self.armed: + return await super().save(data, codec) + + self.started_saves += 1 + save_number = self.started_saves + self.in_flight_saves += 1 + try: + if save_number == self.fail_on_save: + await asyncio.sleep(0) + raise ConnectionError("simulated shard save failure") + await asyncio.sleep(_SAVE_DELAY_SECONDS) + result = await super().save(data, codec) + self.finished_saves += 1 + return result + finally: + self.in_flight_saves -= 1 + + +async def build_wide_hamt(cas: DelayedBytesCAS) -> HAMT: + hamt = await HAMT.build( + cas=cas, + max_bucket_size=1, + values_are_bytes=True, + ) + for index in range(_HAMT_KEY_COUNT): + await hamt.set(f"key-{index}", b"value") + return hamt + + +async def build_dirty_sharded_store(cas: DelayedCIDCAS) -> ShardedZarrStore: + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + chunks_per_shard=1, + ) + await store.set("array/zarr.json", PROTOTYPE.buffer.from_bytes(ARRAY_METADATA)) + for shard_index in range(_SHARD_COUNT): + await store.set( + f"array/c/{shard_index}/0", + PROTOTYPE.buffer.from_bytes(bytes([shard_index])), + ) + return store + + +@pytest.mark.asyncio +async def test_vacate_bounds_simultaneously_live_save_tasks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cas = DelayedBytesCAS() + hamt = await build_wide_hamt(cas) + node_store = cast(InMemoryTreeStore, hamt.node_store) + buffered_node_count = len(node_store.buffer) + assert buffered_node_count > _VACATE_CONCURRENCY + cas.arm() + tracker = TaskCreationTracker() + + with monkeypatch.context() as scoped_monkeypatch: + tracker.install(scoped_monkeypatch) + await hamt.cache_vacate() + + assert tracker.peak_live_tasks <= _VACATE_CONCURRENCY, ( + "vacate must not create the entire save wave as pending tasks: " + f"peak={tracker.peak_live_tasks}, limit={_VACATE_CONCURRENCY}, " + f"buffered_nodes={buffered_node_count}" + ) + + +@pytest.mark.asyncio +async def test_shard_flush_bounds_simultaneously_live_save_tasks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cas = DelayedCIDCAS() + store = await build_dirty_sharded_store(cas) + cas.arm() + tracker = TaskCreationTracker() + + with monkeypatch.context() as scoped_monkeypatch: + tracker.install(scoped_monkeypatch) + await store.flush() + + assert tracker.peak_live_tasks <= _FLUSH_CONCURRENCY, ( + "shard flush must not create every dirty shard as a pending task: " + f"peak={tracker.peak_live_tasks}, limit={_FLUSH_CONCURRENCY}, " + f"dirty_shards={_SHARD_COUNT}" + ) + + +@pytest.mark.asyncio +async def test_failed_vacate_cancels_bounded_work_without_orphans( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cas = DelayedBytesCAS(fail_on_save=3) + hamt = await build_wide_hamt(cas) + total_nodes = len(cast(InMemoryTreeStore, hamt.node_store).buffer) + cas.arm() + tracker = TaskCreationTracker() + + with monkeypatch.context() as scoped_monkeypatch: + tracker.install(scoped_monkeypatch) + with pytest.raises(ConnectionError, match="simulated HAMT save failure"): + await hamt.cache_vacate() + + assert cas.started_saves < total_nodes, "failure must stop unscheduled node saves" + assert cas.in_flight_saves == 0 + assert tracker.pending_tasks() == [], "vacate save tasks survived the failure" + + +@pytest.mark.asyncio +async def test_failed_shard_flush_cancels_bounded_work_without_orphans( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cas = DelayedCIDCAS(fail_on_save=3) + store = await build_dirty_sharded_store(cas) + cas.arm() + tracker = TaskCreationTracker() + + with monkeypatch.context() as scoped_monkeypatch: + tracker.install(scoped_monkeypatch) + with pytest.raises(ConnectionError, match="simulated shard save failure"): + await store.flush() + + assert cas.started_saves < _SHARD_COUNT, "failure must stop unscheduled shard saves" + assert cas.in_flight_saves == 0 + assert tracker.pending_tasks() == [], "shard save tasks survived the failure" diff --git a/tests/test_sharded_zarr_store.py b/tests/test_sharded_zarr_store.py index fb66d30..c4f9d09 100644 --- a/tests/test_sharded_zarr_store.py +++ b/tests/test_sharded_zarr_store.py @@ -266,7 +266,7 @@ async def load_shard(): assert result == shard_data # Verify shard is cached and no pending loads remain - assert await store._shard_data_cache.__contains__(shard_idx) + assert shard_idx in store._shard_data_cache assert await store._shard_data_cache.get(shard_idx) == shard_data assert shard_idx not in store._pending_shard_loads @@ -785,12 +785,15 @@ async def test_listing_and_metadata( prefixed_dir_keys = {key async for key in store_read.list_dir(prefix)} assert {"zarr.json"}.issubset(prefixed_dir_keys) - with pytest.raises( - ValueError, match="Byte range requests are not supported for metadata keys." - ): - proto = zarr.core.buffer.default_buffer_prototype() - byte_range = zarr.abc.store.RangeByteRequest(start=10, end=50) - await store_read.get("lat/zarr.json", proto, byte_range=byte_range) + proto = zarr.core.buffer.default_buffer_prototype() + byte_range = zarr.abc.store.RangeByteRequest(start=10, end=50) + full_metadata = await store_read.get("lat/zarr.json", proto) + ranged_metadata = await store_read.get( + "lat/zarr.json", proto, byte_range=byte_range + ) + assert full_metadata is not None + assert ranged_metadata is not None + assert ranged_metadata.to_bytes() == full_metadata.to_bytes()[10:50] @pytest.mark.asyncio @@ -988,7 +991,8 @@ async def test_sharded_zarr_store_parse_chunk_key(create_ipfs: tuple[str, str]): assert store._parse_chunk_key("lat/c/0/0") is None assert store._parse_chunk_key("lon/c/0/0") is None - # Test dimensionality mismatch + # A wrong-rank named key with no registered metadata for its array + # still fails coordinate validation loudly. with pytest.raises(IndexError, match="tuple index out of range"): store._parse_chunk_key("temp/c/0/0/0/0") @@ -1355,7 +1359,7 @@ async def test_memory_bounded_lru_cache_basic(): # Test basic put/get await cache.put(0, small_shard) assert await cache.get(0) == small_shard - assert await cache.__contains__(0) + assert 0 in cache assert cache.cache_size == 1 # Test that get moves item to end (most recently used) diff --git a/tests/test_sharded_zarr_store_coverage.py b/tests/test_sharded_zarr_store_coverage.py index b42a39e..8036aa6 100644 --- a/tests/test_sharded_zarr_store_coverage.py +++ b/tests/test_sharded_zarr_store_coverage.py @@ -299,6 +299,20 @@ async def test_memory_bounded_lru_cache_update_existing(): assert cache.dirty_cache_size == 1 +@pytest.mark.asyncio +async def test_memory_bounded_lru_cache_atomic_update_guards_and_marks_dirty() -> None: + from py_hamt.sharded_zarr_store import MemoryBoundedLRUCache + + cache = MemoryBoundedLRUCache(max_memory_bytes=10000) + + with pytest.raises(RuntimeError, match="Shard 999 not found in cache"): + await cache.update_entry(999, 0, None) + + await cache.put(0, [None]) + await cache.mark_dirty(0) + assert cache.dirty_cache_size == 1 + + @pytest.mark.asyncio async def test_memory_bounded_lru_cache_eviction_break(): """Test line 96: eviction break when no clean shards available""" diff --git a/tests/test_z10_concurrent_flush.py b/tests/test_z10_concurrent_flush.py new file mode 100644 index 0000000..663597e --- /dev/null +++ b/tests/test_z10_concurrent_flush.py @@ -0,0 +1,122 @@ +import asyncio +import json + +import pytest +import zarr +from multiformats import CID +from testing_utils import CIDInMemoryCAS + +from py_hamt import ContentAddressedStore, ShardedZarrStore + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [12, 2], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2, 2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes"}], + }, + separators=(",", ":"), +).encode() + + +class InstrumentedCAS(CIDInMemoryCAS): + def __init__(self) -> None: + super().__init__() + self.in_flight_saves = 0 + self.max_concurrent_saves = 0 + + async def save(self, data: bytes, codec: ContentAddressedStore.CodecInput) -> CID: + self.in_flight_saves += 1 + self.max_concurrent_saves = max(self.max_concurrent_saves, self.in_flight_saves) + try: + await asyncio.sleep(0.02) + return await super().save(data, codec) + finally: + self.in_flight_saves -= 1 + + def reset_save_concurrency(self) -> None: + self.in_flight_saves = 0 + self.max_concurrent_saves = 0 + + +class FailingCAS(InstrumentedCAS): + def __init__(self, fail_on_save_number: int) -> None: + super().__init__() + self.fail_on_save_number = fail_on_save_number + self.save_calls = 0 + + async def save(self, data: bytes, codec: ContentAddressedStore.CodecInput) -> CID: + self.save_calls += 1 + if self.save_calls == self.fail_on_save_number: + await asyncio.sleep(0.005) + raise ConnectionError("simulated CAS save failure") + return await super().save(data, codec) + + +def buf(data: bytes) -> zarr.core.buffer.Buffer: + return PROTOTYPE.buffer.from_bytes(data) + + +@pytest.mark.asyncio +async def test_dirty_shards_flush_concurrently() -> None: + cas = InstrumentedCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + chunks_per_shard=1, + ) + await store.set("myarr/zarr.json", buf(ARRAY_METADATA)) + for row in range(6): + await store.set(f"myarr/c/{row}/0", buf(bytes([row]))) + + cas.reset_save_concurrency() + await store.flush() + + assert cas.max_concurrent_saves >= 2, "dirty shards must flush concurrently" + + +@pytest.mark.asyncio +async def test_failed_flush_leaves_no_orphan_tasks_and_is_retryable() -> None: + cas = FailingCAS(fail_on_save_number=0) + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + chunks_per_shard=1, + ) + await store.set("myarr/zarr.json", buf(ARRAY_METADATA)) + for row in range(6): + await store.set(f"myarr/c/{row}/0", buf(bytes([row]))) + cas.save_calls = 0 + cas.fail_on_save_number = 3 + + with pytest.raises(ConnectionError): + await store.flush() + + # A failed flush must not leave sibling flush tasks running in the + # background, where they would mutate store state without the write lock. + pending = [ + task + for task in asyncio.all_tasks() + if task is not asyncio.current_task() and not task.done() + ] + assert pending == [], f"orphaned flush tasks survived the failure: {pending}" + + cas.fail_on_save_number = 0 + root_cid = await store.flush() + + reopened = await ShardedZarrStore.open(cas=cas, read_only=True, root_cid=root_cid) + for row in range(6): + chunk = await reopened.get(f"myarr/c/{row}/0", PROTOTYPE) + assert chunk is not None + assert chunk.to_bytes() == bytes([row]) diff --git a/tests/test_z11_v1_unrecorded_coord_chunk.py b/tests/test_z11_v1_unrecorded_coord_chunk.py new file mode 100644 index 0000000..6feba7c --- /dev/null +++ b/tests/test_z11_v1_unrecorded_coord_chunk.py @@ -0,0 +1,218 @@ +import json + +import pytest +import zarr +from hypothesis import given, settings +from hypothesis import strategies as st +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +COORD_ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "attributes": {}, + }, + separators=(",", ":"), +).encode() + + +def buf(data: bytes) -> zarr.core.buffer.Buffer: + return PROTOTYPE.buffer.from_bytes(data) + + +def array_metadata(rank: int) -> bytes: + return json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4] * rank, + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2] * rank}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes"}], + "attributes": {}, + }, + separators=(",", ":"), + ).encode() + + +async def new_v1_store(cas: CIDInMemoryCAS) -> ShardedZarrStore: + return await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + + +async def assert_value(store: ShardedZarrStore, key: str, expected: bytes) -> None: + value = await store.get(key, PROTOTYPE) + assert value is not None + assert value.to_bytes() == expected + + +@pytest.mark.asyncio +async def test_v1_coordinate_chunk_before_primary_chunk_round_trips() -> None: + cas = CIDInMemoryCAS() + store = await new_v1_store(cas) + + await store.set("y/zarr.json", buf(COORD_ARRAY_METADATA)) + await store.set("y/c/0", buf(b"coord-chunk")) + await assert_value(store, "y/c/0", b"coord-chunk") + + await store.set("temp/c/0/0", buf(b"primary")) + await assert_value(store, "temp/c/0/0", b"primary") + + root_cid = await store.flush() + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + await assert_value(reopened, "y/c/0", b"coord-chunk") + + +@pytest.mark.asyncio +async def test_v1_coordinate_chunk_after_primary_chunk_round_trips() -> None: + cas = CIDInMemoryCAS() + store = await new_v1_store(cas) + + await store.set("temp/c/0/0", buf(b"primary")) + await assert_value(store, "temp/c/0/0", b"primary") + + await store.set("y/zarr.json", buf(COORD_ARRAY_METADATA)) + await store.set("y/c/0", buf(b"coord-chunk")) + await assert_value(store, "y/c/0", b"coord-chunk") + + root_cid = await store.flush() + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + await assert_value(reopened, "y/c/0", b"coord-chunk") + + +@pytest.mark.asyncio +async def test_v1_wrong_rank_key_under_recorded_primary_fails_loud() -> None: + """A malformed (wrong-rank) key under the recorded primary path must keep + failing coordinate validation instead of being silently reclassified as + metadata — masking it would let a store opened with a mismatched + array_shape divert every chunk into root metadata.""" + cas = CIDInMemoryCAS() + store = await new_v1_store(cas) + + await store.set("temp/zarr.json", buf(COORD_ARRAY_METADATA)) + await store.set("temp/c/0/0", buf(b"primary")) # records "temp" as primary + + with pytest.raises((IndexError, RuntimeError)): + await store.set("temp/c/0", buf(b"malformed")) + + +@pytest.mark.asyncio +@given( + data=st.data(), + coordinate_path=st.sampled_from(( + "lat", + "lon", + "time", + "group/latitude", + "forecast_reference_time", + )), + coordinate_rank=st.integers(min_value=1, max_value=3), + coordinate_payload=st.binary(max_size=64), + first_primary_payload=st.binary(max_size=64), + second_primary_payload=st.binary(max_size=64), + invalid_primary_rank=st.sampled_from((1, 3)), +) +@settings(max_examples=25, deadline=None) +async def test_v1_unrecorded_coordinate_chunk_ordering_property( + data: st.DataObject, + coordinate_path: str, + coordinate_rank: int, + coordinate_payload: bytes, + first_primary_payload: bytes, + second_primary_payload: bytes, + invalid_primary_rank: int, +) -> None: + """Coordinate chunks remain metadata-backed around primary shard writes.""" + cas = CIDInMemoryCAS() + store = await new_v1_store(cas) + coordinate_key = f"{coordinate_path}/c/" + "/".join( + "0" for _ in range(coordinate_rank) + ) + primary_values = { + "temp/c/0/0": first_primary_payload, + "temp/c/0/1": second_primary_payload, + } + actions = data.draw( + st.permutations(( + "coordinate_metadata", + "coordinate_chunk", + "primary_metadata", + "first_primary_chunk", + "second_primary_chunk", + )), + label="write_order", + ) + written_values: dict[str, bytes] = {} + + for action in actions: + if action == "coordinate_metadata": + await store.set( + f"{coordinate_path}/zarr.json", + buf(array_metadata(coordinate_rank)), + ) + elif action == "coordinate_chunk": + await store.set(coordinate_key, buf(coordinate_payload)) + written_values[coordinate_key] = coordinate_payload + elif action == "primary_metadata": + await store.set("temp/zarr.json", buf(array_metadata(2))) + elif action == "first_primary_chunk": + key = "temp/c/0/0" + await store.set(key, buf(primary_values[key])) + written_values[key] = primary_values[key] + else: + key = "temp/c/0/1" + await store.set(key, buf(primary_values[key])) + written_values[key] = primary_values[key] + + for key, expected in written_values.items(): + await assert_value(store, key, expected) + + invalid_primary_key = "temp/c/" + "/".join("0" for _ in range(invalid_primary_rank)) + with pytest.raises((IndexError, RuntimeError)): + await store.set(invalid_primary_key, buf(b"malformed")) + + root_cid = await store.flush() + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + await assert_value(reopened, coordinate_key, coordinate_payload) + for key, expected in primary_values.items(): + await assert_value(reopened, key, expected) diff --git a/tests/test_z12_v1_legacy_primary_inference.py b/tests/test_z12_v1_legacy_primary_inference.py new file mode 100644 index 0000000..779ccc6 --- /dev/null +++ b/tests/test_z12_v1_legacy_primary_inference.py @@ -0,0 +1,651 @@ +import asyncio +import json +from typing import Any, Literal, cast + +import dag_cbor +import pytest +import zarr +from dag_cbor.ipld import IPLDKind +from hypothesis import example, given, settings +from hypothesis import strategies as st +from hypothesis.internal.conjecture.data import ConjectureData +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore +from py_hamt.sharded_zarr_store import _V1_INFERENCE_CONCURRENCY + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4, 4], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2, 2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "attributes": {}, + }, + separators=(",", ":"), +).encode() +CHUNKLESS_ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4, 4], + }, + separators=(",", ":"), +).encode() + +CandidateFormat = Literal["v2", "v3", "dual"] +CandidateCondition = Literal[ + "valid", + "undecodable", + "non_dict", + "wrong_shape", + "missing_chunk_shape", + "wrong_chunk_shape", +] +FailureKind = Literal["none", "coordinate", "real"] +CandidateSpec = tuple[str, CandidateFormat, CandidateCondition] +CandidateDocument = tuple[str, bytes, str, bool] + +CANDIDATE_FORMATS: tuple[CandidateFormat, ...] = ("v2", "v3", "dual") +CANDIDATE_CONDITIONS: tuple[CandidateCondition, ...] = ( + "valid", + "undecodable", + "non_dict", + "wrong_shape", + "missing_chunk_shape", + "wrong_chunk_shape", +) +COORDINATE_NAMES = ( + "lat", + "lon", + "time", + "latitude", + "longitude", + "forecast_reference_time", + "step", +) +COORDINATE_PATHS = ("lat", "/lon", "group/time", *COORDINATE_NAMES[3:]) + + +def buf(data: bytes) -> zarr.core.buffer.Buffer: + return PROTOTYPE.buffer.from_bytes(data) + + +def decode_root(data: bytes) -> dict[str, Any]: + root_obj = dag_cbor.decode(data) + assert isinstance(root_obj, dict) + return cast(dict[str, Any], root_obj) + + +def _candidate_payload( + path: str, + metadata_format: Literal["v2", "v3"], + condition: CandidateCondition, +) -> bytes: + if condition == "undecodable": + return b"\xffnot-json:" + path.encode() + if condition == "non_dict": + return json.dumps([path, metadata_format]).encode() + + metadata: dict[str, object] = { + "candidate": path, + "zarr_format": 2 if metadata_format == "v2" else 3, + "shape": [8, 8] if condition == "wrong_shape" else [4, 4], + } + if condition != "missing_chunk_shape": + declared_chunks = [4, 4] if condition == "wrong_chunk_shape" else [2, 2] + if metadata_format == "v2": + metadata["chunks"] = declared_chunks + else: + metadata["chunk_grid"] = { + "name": "regular", + "configuration": {"chunk_shape": declared_chunks}, + } + return json.dumps(metadata, separators=(",", ":")).encode() + + +def _candidate_documents(spec: CandidateSpec) -> list[CandidateDocument]: + path, metadata_format, condition = spec + normalized_path = path.strip("/") + coordinate = normalized_path.rsplit("/", 1)[-1] in COORDINATE_NAMES + formats: tuple[Literal["v2", "v3"], ...] = ( + ("v2", "v3") if metadata_format == "dual" else (metadata_format,) + ) + return [ + ( + f"{path}/.zarray" if document_format == "v2" else f"{path}/zarr.json", + _candidate_payload(path, document_format, condition), + normalized_path, + coordinate, + ) + for document_format in formats + ] + + +def _pinned_data() -> st.DataObject: + """Return deterministic draws for explicit examples using ``st.data()``.""" + return st.DataObject(ConjectureData.for_choices([0] * 8)) + + +@pytest.mark.asyncio +async def test_v1_legacy_root_infers_primary_array_path_for_listing() -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/zarr.json", buf(ARRAY_METADATA)) + await store.set("temp/c/0/0", buf(b"chunk")) + root = await store.flush() + + root_obj = decode_root(await cas.load(root)) + root_obj["chunks"].pop("primary_array_path") + legacy_root = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root), + ) + + chunk = await reopened.get("temp/c/0/0", PROTOTYPE) + assert chunk is not None + assert chunk.to_bytes() == b"chunk" + + listed = [key async for key in reopened.list()] + assert "temp/c/0/0" in listed and "c/0/0" not in listed + assert [key async for key in reopened.list_prefix("temp/c/")] == ["temp/c/0/0"] + + +@pytest.mark.asyncio +async def test_inference_rejects_lone_chunkless_candidate() -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/zarr.json", buf(ARRAY_METADATA)) + await store.set("temp/c/0/0", buf(b"chunk")) + root = await store.flush() + + root_obj = decode_root(await cas.load(root)) + root_obj["chunks"].pop("primary_array_path") + malformed_cid = await cas.save(CHUNKLESS_ARRAY_METADATA, codec="raw") + root_obj["metadata"] = {"malformed/zarr.json": malformed_cid} + legacy_root = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root), + ) + assert reopened._primary_array_path in (None, "") + + +@pytest.mark.asyncio +async def test_inference_ignores_chunkless_candidate_beside_valid_array() -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/zarr.json", buf(ARRAY_METADATA)) + await store.set("temp/c/0/0", buf(b"chunk")) + root = await store.flush() + + root_obj = decode_root(await cas.load(root)) + root_obj["chunks"].pop("primary_array_path") + root_obj["metadata"]["malformed/zarr.json"] = await cas.save( + CHUNKLESS_ARRAY_METADATA, + codec="raw", + ) + legacy_root = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root), + ) + assert reopened._primary_array_path == "temp" + + +class FlakyMetadataCAS(CIDInMemoryCAS): + """Fails loads of one specific CID to simulate a partial CAS outage.""" + + def __init__(self) -> None: + super().__init__() + self.failing_cid: object = None + + async def load( + self, + identifier: IPLDKind, + offset: int | None = None, + length: int | None = None, + suffix: int | None = None, + ) -> bytes: + if self.failing_cid is not None and str(identifier) == str(self.failing_cid): + raise ConnectionError("simulated partial CAS outage") + return await super().load( + identifier, + offset=offset, + length=length, + suffix=suffix, + ) + + +class ConcurrentMetadataCAS(CIDInMemoryCAS): + """Measure concurrency and request count for selected metadata loads.""" + + def __init__(self) -> None: + super().__init__() + self.delayed_cids: list[IPLDKind] = [] + self.load_count = 0 + self.in_flight = 0 + self.max_in_flight = 0 + + async def load( + self, + identifier: IPLDKind, + offset: int | None = None, + length: int | None = None, + suffix: int | None = None, + ) -> bytes: + if not any(identifier == candidate for candidate in self.delayed_cids): + return await super().load( + identifier, + offset=offset, + length=length, + suffix=suffix, + ) + + self.load_count += 1 + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + try: + await asyncio.sleep(0.01) + return await super().load( + identifier, + offset=offset, + length=length, + suffix=suffix, + ) + finally: + self.in_flight -= 1 + + +@pytest.mark.asyncio +async def test_inference_aborts_when_a_candidate_cannot_be_loaded() -> None: + """If any candidate's metadata cannot be loaded, the true primary might be + the unreadable one — inference must abort (keep the legacy '' default) + rather than confidently adopt a surviving same-shape candidate.""" + cas = FlakyMetadataCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/zarr.json", buf(ARRAY_METADATA)) + await store.set("precip/zarr.json", buf(ARRAY_METADATA)) + await store.set("temp/c/0/0", buf(b"chunk")) + root = await store.flush() + + root_obj = decode_root(await cas.load(root)) + root_obj["chunks"].pop("primary_array_path") + legacy_root = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + + # Make temp's metadata unreadable: only "precip" would survive the scan. + metadata = root_obj["metadata"] + cas.failing_cid = metadata["temp/zarr.json"] + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root), + ) + assert reopened._primary_array_path in (None, ""), ( + "a partial CAS outage must not rebind shard data under a surviving " + f"candidate, got {reopened._primary_array_path!r}" + ) + + +@pytest.mark.asyncio +async def test_inference_loads_candidates_concurrently_and_stops_when_ambiguous() -> ( + None +): + cas = ConcurrentMetadataCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/c/0/0", buf(b"chunk")) + root_cid = await store.flush() + root_obj = decode_root(await cas.load(root_cid)) + root_obj["chunks"].pop("primary_array_path") + + candidate_count = _V1_INFERENCE_CONCURRENCY * 3 + metadata: dict[str, IPLDKind] = {} + for index in range(candidate_count): + candidate_cid = await cas.save( + json.dumps({ + "candidate": index, + "shape": [4, 4], + "chunks": [2, 2], + }).encode(), + codec="raw", + ) + metadata[f"array-{index}/.zarray"] = candidate_cid + root_obj["metadata"] = metadata + legacy_root_cid = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + cas.delayed_cids = list(metadata.values()) + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root_cid), + ) + + assert reopened._primary_array_path == "" + assert cas.max_in_flight == _V1_INFERENCE_CONCURRENCY + assert cas.load_count == _V1_INFERENCE_CONCURRENCY + assert cas.load_count < candidate_count + + +@pytest.mark.asyncio +async def test_inference_propagates_candidate_load_cancellation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + store._root_obj["chunks"].pop("primary_array_path", None) + metadata_cid = await cas.save(ARRAY_METADATA, codec="raw") + store._root_obj["metadata"] = {"temp/zarr.json": metadata_cid} + original_load = cas.load + + async def cancelling_load( + identifier: IPLDKind, + offset: int | None = None, + length: int | None = None, + suffix: int | None = None, + ) -> bytes: + if identifier == metadata_cid: + raise asyncio.CancelledError + return await original_load( + identifier, + offset=offset, + length=length, + suffix=suffix, + ) + + monkeypatch.setattr(cas, "load", cancelling_load) + + with pytest.raises(asyncio.CancelledError): + await store._infer_v1_legacy_primary_array_path() + + +@pytest.mark.asyncio +async def test_inference_skips_non_candidates_and_dedupes_dual_format() -> None: + """The scan must skip coordinate arrays, undecodable blobs, shape and + chunk-shape mismatches, non-string keys, and count an array once even + when it registers both zarr.json and v2 .zarray metadata.""" + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/zarr.json", buf(ARRAY_METADATA)) + await store.set("temp/c/0/0", buf(b"chunk")) + root = await store.flush() + + def meta(payload: dict[str, object]) -> bytes: + return json.dumps(payload).encode() + + root_obj = decode_root(await cas.load(root)) + root_obj["chunks"].pop("primary_array_path") + metadata = root_obj["metadata"] + metadata["lat/zarr.json"] = await cas.save( + meta({ + "shape": [4, 4], + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2, 2]}, + }, + }), + codec="raw", + ) + metadata["garbage/zarr.json"] = await cas.save(b"not-json", codec="raw") + metadata["wrongshape/zarr.json"] = await cas.save( + meta({"shape": [8, 8]}), codec="raw" + ) + metadata["wrongchunks/zarr.json"] = await cas.save( + meta({ + "shape": [4, 4], + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [4, 4]}, + }, + }), + codec="raw", + ) + # v2-format dual registration of the same array: "chunks" key, same grid. + metadata["temp/.zarray"] = await cas.save( + meta({"shape": [4, 4], "chunks": [2, 2]}), codec="raw" + ) + legacy_root = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root), + ) + assert reopened._primary_array_path == "temp" + listed = [key async for key in reopened.list()] + assert "temp/c/0/0" in listed + + +@pytest.mark.asyncio +async def test_inference_handles_malformed_metadata_maps() -> None: + """Direct-call coverage for defensive branches dag-cbor cannot produce: + a non-mapping metadata value and a non-string metadata key.""" + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + store._root_obj["chunks"].pop("primary_array_path", None) + + store._root_obj["metadata"] = None + await store._infer_v1_legacy_primary_array_path() + assert store._primary_array_path in (None, "") + + cid = await cas.save(b"payload", codec="raw") + store._root_obj["metadata"] = {1: cid} + await store._infer_v1_legacy_primary_array_path() + assert store._primary_array_path in (None, "") + + +@pytest.mark.asyncio +@example( + data=_pinned_data(), + first_path="temp", + first_format="v3", + first_condition="valid", + second_path="precip", + second_format="v2", + second_condition="wrong_shape", + coordinate_path="lat", + coordinate_format="v3", + coordinate_condition="valid", + failure_kind="none", +) +@example( + data=_pinned_data(), + first_path="temp", + first_format="v2", + first_condition="valid", + second_path="precip", + second_format="v3", + second_condition="wrong_chunk_shape", + coordinate_path="/lon", + coordinate_format="v2", + coordinate_condition="valid", + failure_kind="coordinate", +) +@given( + data=st.data(), + first_path=st.sampled_from(("temp", "/temp", "group/temp")), + first_format=st.sampled_from(CANDIDATE_FORMATS), + first_condition=st.sampled_from(CANDIDATE_CONDITIONS), + second_path=st.sampled_from(("precip", "/precip", "group/precip")), + second_format=st.sampled_from(CANDIDATE_FORMATS), + second_condition=st.sampled_from(CANDIDATE_CONDITIONS), + coordinate_path=st.sampled_from(COORDINATE_PATHS), + coordinate_format=st.sampled_from(CANDIDATE_FORMATS), + coordinate_condition=st.sampled_from(CANDIDATE_CONDITIONS), + failure_kind=st.sampled_from(("none", "coordinate", "real")), +) +@settings(max_examples=25, deadline=None) +async def test_v1_legacy_primary_inference_property( + data: st.DataObject, + first_path: str, + first_format: CandidateFormat, + first_condition: CandidateCondition, + second_path: str, + second_format: CandidateFormat, + second_condition: CandidateCondition, + coordinate_path: str, + coordinate_format: CandidateFormat, + coordinate_condition: CandidateCondition, + failure_kind: FailureKind, +) -> None: + """Inference is order-independent and conservative. + + An unambiguous result is recorded on a writable open so a later reopen is + deterministic; an ambiguous or absent result records nothing. + """ + cas = FlakyMetadataCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/c/0/0", buf(b"chunk")) + root_cid = await store.flush() + root_obj = decode_root(await cas.load(root_cid)) + root_obj["chunks"].pop("primary_array_path") + + specs: tuple[CandidateSpec, ...] = ( + (first_path, first_format, first_condition), + (second_path, second_format, second_condition), + (coordinate_path, coordinate_format, coordinate_condition), + ) + documents = [document for spec in specs for document in _candidate_documents(spec)] + ordered_documents = data.draw( + st.permutations(documents), + label="metadata_insertion_order", + ) + + metadata: dict[str, IPLDKind] = {} + saved_documents: list[tuple[CandidateDocument, IPLDKind]] = [] + for document in ordered_documents: + key, payload, _, _ = document + cid = await cas.save(payload, codec="raw") + metadata[key] = cid + saved_documents.append((document, cid)) + root_obj["metadata"] = metadata + + failing_cid: IPLDKind | None = None + if failure_kind != "none": + eligible_documents = [ + saved + for saved in saved_documents + if saved[0][3] is (failure_kind == "coordinate") + ] + _, failing_cid = data.draw( + st.sampled_from(eligible_documents), + label=f"{failure_kind}_load_failure_position", + ) + + original_chunks = dict(root_obj["chunks"]) + legacy_root = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=str(legacy_root), + ) + # DAG-CBOR canonicalizes map ordering. Restore the generated order for the + # direct inference seam and reset the artifacts the writable open's own + # inference already produced, so this call is exercised in isolation. + reopened._root_obj["metadata"] = metadata + reopened._root_obj["chunks"] = dict(original_chunks) + reopened._primary_array_path = "" + reopened._primary_inferred = False + reopened._dirty_root = False + cas.failing_cid = failing_cid + await reopened._infer_v1_legacy_primary_array_path() + + valid_paths = { + path.strip("/") + for path, _, condition in specs + if condition == "valid" + and path.strip("/").rsplit("/", 1)[-1] not in COORDINATE_NAMES + } + expected_path = ( + next(iter(valid_paths)) + if failure_kind != "real" and len(valid_paths) == 1 + else "" + ) + assert reopened._primary_array_path == expected_path + + if expected_path: + # Unambiguous inference on a writable open is recorded so a later reopen + # routes from the recorded primary instead of re-inferring. + expected_chunks = {**original_chunks, "primary_array_path": expected_path} + assert reopened._root_obj["chunks"] == expected_chunks + flushed_root = await reopened.flush() + assert flushed_root != str(legacy_root) + assert decode_root(await cas.load(flushed_root))["chunks"] == expected_chunks + else: + # An ambiguous or absent result records nothing. + assert reopened._root_obj["chunks"] == original_chunks + flushed_root = await reopened.flush() + assert flushed_root == str(legacy_root) + assert decode_root(await cas.load(flushed_root))["chunks"] == original_chunks diff --git a/tests/test_z13_inferred_primary_write_stability.py b/tests/test_z13_inferred_primary_write_stability.py new file mode 100644 index 0000000..594ad1a --- /dev/null +++ b/tests/test_z13_inferred_primary_write_stability.py @@ -0,0 +1,624 @@ +import json +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, cast + +import dag_cbor +import pytest +import zarr +from dag_cbor.ipld import IPLDKind +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +TEMP_CHUNKS = { + "temp/c/0/0": b"temp-00", + "temp/c/0/1": b"temp-01", + "temp/c/1/0": b"temp-10", + "temp/c/1/1": b"temp-11", +} +PRECIP_KEY = "precip/c/0/0" +PRECIP_CHUNK = b"precip-00" +FOREIGN_SAME_RANK_KEY = "precip/c/0/0" +FOREIGN_SAME_RANK_CHUNK = b"precip-same-rank-00" + + +def array_metadata(*, shape: tuple[int, ...], chunk_shape: tuple[int, ...]) -> bytes: + return json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": list(shape), + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": list(chunk_shape)}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "attributes": {}, + }, + separators=(",", ":"), + ).encode() + + +def buf(data: bytes) -> zarr.core.buffer.Buffer: + return PROTOTYPE.buffer.from_bytes(data) + + +class FailingShardLoadCAS(CIDInMemoryCAS): + """Fail one selected load after a legacy root has been opened.""" + + def __init__(self) -> None: + super().__init__() + self.failing_cid: IPLDKind | None = None + + async def load( + self, + identifier: IPLDKind, + offset: int | None = None, + length: int | None = None, + suffix: int | None = None, + ) -> bytes: + if self.failing_cid is not None and str(identifier) == str(self.failing_cid): + raise RuntimeError("simulated shard load failure") + return await super().load( + identifier, + offset=offset, + length=length, + suffix=suffix, + ) + + +async def decoded_root(cas: CIDInMemoryCAS, root_cid: str) -> dict[str, Any]: + root_obj = dag_cbor.decode(await cas.load(root_cid)) + assert isinstance(root_obj, dict) + return cast(dict[str, Any], root_obj) + + +async def v1_recorded_and_legacy_roots() -> tuple[CIDInMemoryCAS, str, str]: + """Build equivalent V1 roots with and without the recorded primary field.""" + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set( + "temp/zarr.json", + buf(array_metadata(shape=(4, 4), chunk_shape=(2, 2))), + ) + # This is metadata for another array, but its geometry excludes it from + # legacy-primary inference. Thus "temp" remains the unique candidate. + await store.set( + "precip/zarr.json", + buf(array_metadata(shape=(4, 4), chunk_shape=(1, 1))), + ) + for key, data in TEMP_CHUNKS.items(): + await store.set(key, buf(data)) + recorded_root_cid = await store.flush() + + root_obj = await decoded_root(cas, recorded_root_cid) + chunk_info = root_obj["chunks"] + assert isinstance(chunk_info, dict) + assert chunk_info["primary_array_path"] == "temp" + chunk_info.pop("primary_array_path") + legacy_root_cid = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + return cas, recorded_root_cid, str(legacy_root_cid) + + +async def read_bytes(store: ShardedZarrStore, key: str) -> bytes | None: + value = await store.get(key, PROTOTYPE) + return None if value is None else value.to_bytes() + + +async def read_temp_chunks(store: ShardedZarrStore) -> dict[str, bytes | None]: + return {key: await read_bytes(store, key) for key in TEMP_CHUNKS} + + +@dataclass(frozen=True) +class NonPrimaryWriteOutcome: + live_primary: str | None + persisted_primary: str | None + precip_in_metadata: bool + temp_chunks: Mapping[str, bytes | None] + precip_chunk: bytes | None + + +async def exercise_non_primary_write( + cas: CIDInMemoryCAS, root_cid: str +) -> NonPrimaryWriteOutcome: + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=root_cid, + ) + assert store._primary_array_path == "temp" + + await store.set(PRECIP_KEY, buf(PRECIP_CHUNK)) + live_primary = store._primary_array_path + flushed_root_cid = await store.flush() + root_obj = await decoded_root(cas, flushed_root_cid) + chunk_info = root_obj["chunks"] + metadata = root_obj["metadata"] + assert isinstance(chunk_info, dict) + assert isinstance(metadata, dict) + persisted_primary = chunk_info.get("primary_array_path") + assert persisted_primary is None or isinstance(persisted_primary, str) + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + return NonPrimaryWriteOutcome( + live_primary=live_primary, + persisted_primary=persisted_primary, + precip_in_metadata=PRECIP_KEY in metadata, + temp_chunks=await read_temp_chunks(reopened), + precip_chunk=await read_bytes(reopened, PRECIP_KEY), + ) + + +async def assert_inferred_primary_write_is_accepted( + cas: CIDInMemoryCAS, legacy_root_cid: str +) -> None: + """A writable open records the unambiguous inferred primary and keeps chunks. + + Recording makes routing deterministic: a later reopen reads from the sealed + primary instead of re-inferring, so a second same-geometry array can no + longer make the reopen inference ambiguous and misroute a chunk. + """ + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=legacy_root_cid, + ) + assert store._primary_array_path == "temp" + + replacement = b"updated-temp-11" + await store.set("temp/c/1/1", buf(replacement)) + assert store._primary_array_path == "temp" + flushed_root_cid = await store.flush() + root_obj = await decoded_root(cas, flushed_root_cid) + chunk_info = root_obj["chunks"] + assert isinstance(chunk_info, dict) + assert chunk_info["primary_array_path"] == "temp" + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + expected_chunks = {**TEMP_CHUNKS, "temp/c/1/1": replacement} + assert await read_temp_chunks(reopened) == expected_chunks + + +@pytest.mark.asyncio +async def test_non_primary_write_matches_recorded_primary_semantics() -> None: + cas, recorded_root_cid, legacy_root_cid = await v1_recorded_and_legacy_roots() + + # A writable open records the unambiguous inferred primary, so a legacy root + # behaves identically to one that already had the primary recorded. + await assert_inferred_primary_write_is_accepted(cas, legacy_root_cid) + + expected = NonPrimaryWriteOutcome( + live_primary="temp", + persisted_primary="temp", + precip_in_metadata=True, + temp_chunks=TEMP_CHUNKS, + precip_chunk=PRECIP_CHUNK, + ) + recorded = await exercise_non_primary_write(cas, recorded_root_cid) + inferred = await exercise_non_primary_write(cas, legacy_root_cid) + assert recorded == expected + assert inferred == expected + + +@pytest.mark.asyncio +async def test_non_primary_write_keeps_every_temp_chunk_readable_after_reopen() -> None: + cas, _, legacy_root_cid = await v1_recorded_and_legacy_roots() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=legacy_root_cid, + ) + assert store._primary_array_path == "temp" + + await store.set(PRECIP_KEY, buf(PRECIP_CHUNK)) + flushed_root_cid = await store.flush() + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + + assert await read_temp_chunks(reopened) == TEMP_CHUNKS + + +@pytest.mark.asyncio +async def test_recorded_empty_primary_routes_named_chunk_to_metadata() -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + root_chunk_key = "/c/0/0" + original_root_chunk = b"root-00" + await store.set(root_chunk_key, buf(original_root_chunk)) + root_cid = await store.flush() + + initial_root = await decoded_root(cas, root_cid) + initial_chunk_info = initial_root["chunks"] + assert isinstance(initial_chunk_info, dict) + assert initial_chunk_info["primary_array_path"] == "" + + reopened_for_write = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=root_cid, + ) + named_chunk_key = "named/c/0/0" + await reopened_for_write.set(named_chunk_key, buf(b"named-00")) + mutated_root_cid = await reopened_for_write.flush() + + mutated_root = await decoded_root(cas, mutated_root_cid) + mutated_chunk_info = mutated_root["chunks"] + mutated_metadata = mutated_root["metadata"] + assert isinstance(mutated_chunk_info, dict) + assert isinstance(mutated_metadata, dict) + + reopened_for_read = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=mutated_root_cid, + ) + assert await read_bytes(reopened_for_read, root_chunk_key) == original_root_chunk + assert named_chunk_key in mutated_metadata + assert mutated_chunk_info["primary_array_path"] == "" + + +@pytest.mark.asyncio +async def test_recorded_empty_primary_listing_round_trips_through_get() -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + root_chunk_key = "/c/0/0" + canonical_root_chunk_key = "c/0/0" + root_chunk = b"root-00" + await store.set(root_chunk_key, buf(root_chunk)) + root_cid = await store.flush() + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + listed = [key async for key in reopened.list()] + + assert listed == [canonical_root_chunk_key] + listed_value = await reopened.get(listed[0], PROTOTYPE) + assert listed_value is not None + assert listed_value.to_bytes() == root_chunk + + root_obj = await decoded_root(cas, root_cid) + chunk_info = root_obj["chunks"] + assert isinstance(chunk_info, dict) + chunk_info.pop("primary_array_path") + legacy_root_cid = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + legacy_reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root_cid), + ) + legacy_listed = [key async for key in legacy_reopened.list()] + assert legacy_listed == [root_chunk_key] + legacy_value = await legacy_reopened.get(legacy_listed[0], PROTOTYPE) + assert legacy_value is not None + assert legacy_value.to_bytes() == root_chunk + + +@pytest.mark.asyncio +async def test_failed_first_primary_write_does_not_persist_its_path() -> None: + cas = FailingShardLoadCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + original_key = "temp/c/0/0" + original_chunk = b"original-temp" + await store.set(original_key, buf(original_chunk)) + recorded_root_cid = await store.flush() + + root_obj = await decoded_root(cas, recorded_root_cid) + chunk_info = root_obj["chunks"] + assert isinstance(chunk_info, dict) + shard_cid = chunk_info["shard_cids"][0] + chunk_info.pop("primary_array_path") + legacy_root_cid = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=str(legacy_root_cid), + ) + cas.failing_cid = shard_cid + with pytest.raises(RuntimeError, match="simulated shard load failure"): + await reopened.set("wrong/c/0/0", buf(b"failed-write")) + + assert "primary_array_path" not in reopened._root_obj["chunks"] + assert reopened._primary_array_path == "" + + cas.failing_cid = None + flushed_root_cid = await reopened.flush() + persisted_root = await decoded_root(cas, flushed_root_cid) + assert "primary_array_path" not in persisted_root["chunks"] + + read_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + assert await read_bytes(read_store, original_key) == original_chunk + + +@pytest.mark.asyncio +async def test_same_geometry_secondary_write_is_routed_deterministically() -> None: + """A second same-geometry array must not corrupt the primary across a reopen. + + Because the writable open records the inferred primary, the reopen routes + from that recorded value instead of re-inferring. Without recording, the + second array's metadata made the reopen inference ambiguous, so its + metadata-stored chunk re-parsed against the shared shard index and read the + primary's slot (``temp-00``) instead of its own bytes. Recording makes the + secondary chunk read back its own value. + """ + cas, _, legacy_root_cid = await v1_recorded_and_legacy_roots() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=legacy_root_cid, + ) + assert store._primary_array_path == "temp" + + await store.set( + "precip2/zarr.json", + buf(array_metadata(shape=(4, 4), chunk_shape=(2, 2))), + ) + secondary_key = "precip2/c/0/0" + secondary_chunk = b"precip2-00" + await store.set(secondary_key, buf(secondary_chunk)) + + assert store._primary_array_path == "temp" + assert await read_temp_chunks(store) == TEMP_CHUNKS + flushed_root_cid = await store.flush() + root_obj = await decoded_root(cas, flushed_root_cid) + chunk_info = root_obj["chunks"] + metadata = root_obj["metadata"] + assert isinstance(chunk_info, dict) + assert isinstance(metadata, dict) + assert secondary_key in metadata + # The inferred primary is now recorded, so the reopen is unambiguous. + assert chunk_info["primary_array_path"] == "temp" + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + assert await read_temp_chunks(reopened) == TEMP_CHUNKS + # The corruption fix: the secondary reads its OWN bytes, not temp's slot. + assert await read_bytes(reopened, secondary_key) == secondary_chunk + + +@pytest.mark.asyncio +async def test_metadata_routed_foreign_chunk_lifecycle_preserves_primary() -> None: + cas, _, legacy_root_cid = await v1_recorded_and_legacy_roots() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=legacy_root_cid, + ) + assert store._primary_array_path == "temp" + + await store.set(PRECIP_KEY, buf(PRECIP_CHUNK)) + assert await store.exists(PRECIP_KEY) + assert await read_bytes(store, PRECIP_KEY) == PRECIP_CHUNK + assert await read_temp_chunks(store) == TEMP_CHUNKS + + await store.delete(PRECIP_KEY) + assert not await store.exists(PRECIP_KEY) + assert await read_bytes(store, PRECIP_KEY) is None + assert await read_temp_chunks(store) == TEMP_CHUNKS + + flushed_root_cid = await store.flush() + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + assert not await reopened.exists(PRECIP_KEY) + assert await read_bytes(reopened, PRECIP_KEY) is None + assert await read_temp_chunks(reopened) == TEMP_CHUNKS + + +async def v1_root_primary_legacy_root() -> tuple[CIDInMemoryCAS, str]: + """A legacy V1 root whose primary infers to the root array itself (path ""). + + The root array's own metadata makes "" the unique inference candidate. No + ``primary_array_path`` is ever sealed (no primary chunk is written), so the + flushed root is legacy-shaped and reopening re-infers "". + """ + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("zarr.json", buf(array_metadata(shape=(4, 4), chunk_shape=(2, 2)))) + root_cid = await store.flush() + + root_obj = await decoded_root(cas, root_cid) + chunk_info = root_obj["chunks"] + assert isinstance(chunk_info, dict) + assert "primary_array_path" not in chunk_info + return cas, root_cid + + +@pytest.mark.asyncio +async def test_inferred_root_primary_is_recorded_and_not_rebound() -> None: + """A foreign same-rank chunk must not rebind an inferred root primary (""). + + Inference sets the primary to the root array (""). A writable open records + that unambiguous inference — even the empty-string path — so the foreign + write routes to metadata and the reopen reads from the recorded primary + rather than re-inferring. This keeps the root shard slot intact and the + foreign chunk readable from its own metadata entry. + """ + cas, legacy_root_cid = await v1_root_primary_legacy_root() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=legacy_root_cid, + ) + assert store._primary_array_path == "" + assert store._primary_inferred is True + + await store.set(FOREIGN_SAME_RANK_KEY, buf(FOREIGN_SAME_RANK_CHUNK)) + + # The inferred root primary is not rebound in memory and is recorded to disk. + assert store._primary_array_path == "" + flushed_root_cid = await store.flush() + root_obj = await decoded_root(cas, flushed_root_cid) + chunk_info = root_obj["chunks"] + metadata = root_obj["metadata"] + assert isinstance(chunk_info, dict) + assert isinstance(metadata, dict) + assert chunk_info["primary_array_path"] == "" + assert FOREIGN_SAME_RANK_KEY in metadata + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + assert reopened._primary_array_path == "" + # The reopen routes from the recorded primary; it does not re-infer. + assert reopened._primary_inferred is False + assert await read_bytes(reopened, FOREIGN_SAME_RANK_KEY) == FOREIGN_SAME_RANK_CHUNK + + +@pytest.mark.asyncio +async def test_read_only_to_writable_clone_seals_inferred_primary() -> None: + """with_read_only(False) must persist a primary inferred under a RO open. + + A read-only open infers the primary but never reaches the persistence + branch (gated on ``not self.read_only``), so the path lives only in the + in-memory ``_primary_inferred`` flag. A writable clone inherits that flag + while sharing an unrecorded root; its first chunk write then skips its own + seal (gated on ``not self._primary_inferred``), so a same-geometry secondary + array could flush with no recorded primary and misroute on reopen. The + clone must perform the deferred seal so it behaves like a writable open. + """ + cas, _, legacy_root_cid = await v1_recorded_and_legacy_roots() + + read_only = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=legacy_root_cid, + ) + assert read_only._primary_array_path == "temp" + assert read_only._primary_inferred is True + # The read-only open never records the inference. + assert "primary_array_path" not in read_only._root_obj["chunks"] + + writable = read_only.with_read_only(False) + assert writable.read_only is False + # The clone seals the inferred primary so routing is deterministic on reopen. + assert writable._root_obj["chunks"]["primary_array_path"] == "temp" + assert writable._dirty_root is True + + await writable.set( + "precip2/zarr.json", + buf(array_metadata(shape=(4, 4), chunk_shape=(2, 2))), + ) + secondary_key = "precip2/c/0/0" + secondary_chunk = b"precip2-00" + await writable.set(secondary_key, buf(secondary_chunk)) + assert writable._primary_array_path == "temp" + + flushed_root_cid = await writable.flush() + root_obj = await decoded_root(cas, flushed_root_cid) + chunk_info = root_obj["chunks"] + metadata = root_obj["metadata"] + assert isinstance(chunk_info, dict) + assert isinstance(metadata, dict) + assert chunk_info["primary_array_path"] == "temp" + assert secondary_key in metadata + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=flushed_root_cid, + ) + assert await read_temp_chunks(reopened) == TEMP_CHUNKS + # The corruption fix: the secondary reads its OWN bytes, not temp's slot. + assert await read_bytes(reopened, secondary_key) == secondary_chunk + + +@pytest.mark.asyncio +async def test_unrecorded_empty_primary_list_dir_is_consistent() -> None: + """list_dir("c") must surface chunks for an unrecorded empty-primary root. + + Such a root emits its shard tree with a leading slash ("/c/...") to keep it + distinct from legacy metadata keys, but list_dir prefixes are slash-stripped. + list_dir("") advertises "c", so descending into list_dir("c") must return the + chunk components rather than an empty listing. + """ + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("/c/0/0", buf(b"root-00")) + root_cid = await store.flush() + + # Drop the sealed primary to model a legacy, unrecorded-primary root. + root_obj = await decoded_root(cas, root_cid) + chunk_info = root_obj["chunks"] + assert isinstance(chunk_info, dict) + chunk_info.pop("primary_array_path") + legacy_root_cid = await cas.save(dag_cbor.encode(root_obj), codec="dag-cbor") + + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=str(legacy_root_cid), + ) + assert reopened._primary_array_path == "" + top_level = {key async for key in reopened.list_dir("")} + assert "c" in top_level + assert [key async for key in reopened.list_dir("c")] == ["0"] diff --git a/tests/test_z14_resize_shard_pin.py b/tests/test_z14_resize_shard_pin.py new file mode 100644 index 0000000..fb61d77 --- /dev/null +++ b/tests/test_z14_resize_shard_pin.py @@ -0,0 +1,49 @@ +import numpy as np +import pytest +import xarray as xr +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore + + +@pytest.mark.asyncio +async def test_resize_snapshot_pins_freshly_loaded_shard() -> None: + """A freshly-fetched resize shard must be pinned against concurrent eviction. + + With an over-budget cache whose other entries are dirty (so unevictable), + the just-loaded clean shard is the only eviction candidate. Without a pin it + is dropped between ``put`` and the follow-up ``get`` inside the snapshot, + raising a spurious ``RuntimeError``. Pinning across fetch+read prevents it. + """ + ds = xr.Dataset( + {"temp": (["x"], np.arange(4.0))}, + coords={"x": np.arange(4)}, + ).chunk({"x": 4}) + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4,), + chunk_shape=(4,), + chunks_per_shard=1, + ) + ds.to_zarr(store=store, mode="w") + root_cid = await store.flush() + + store = await ShardedZarrStore.open(cas=cas, read_only=False, root_cid=root_cid) + array_index = store.array_indices[""] + assert array_index.num_shards >= 1 + assert array_index.shard_cids[0] is not None + + cache = store._shard_data_cache + await cache.clear() + # Squeeze the budget and keep a dirty (unevictable) entry resident so the + # cache is permanently over budget: the snapshot's fresh shard becomes the + # sole eviction candidate. + cache.max_memory_bytes = 1 + await cache.put(9999, [None], is_dirty=True) + + snapshot = await store._snapshot_shards_for_resize(array_index) + + assert 0 in snapshot + assert len(snapshot[0]) == array_index.chunks_per_shard diff --git a/tests/test_z15_v1_list_dir_chunk_descent.py b/tests/test_z15_v1_list_dir_chunk_descent.py new file mode 100644 index 0000000..31d5145 --- /dev/null +++ b/tests/test_z15_v1_list_dir_chunk_descent.py @@ -0,0 +1,53 @@ +import numpy as np +import pytest +import xarray as xr +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore + + +@pytest.mark.asyncio +async def test_v1_list_dir_descends_into_chunk_directory() -> None: + """V1 ``list_dir("/c")`` must enumerate coordinate directories. + + Previously only V2 recognised a chunk prefix, so a V1 store reported "c" + under the primary but returned nothing when listed one level deeper. + """ + ds = xr.Dataset( + {"temp": (["y", "x"], np.arange(16.0).reshape(4, 4))}, + coords={"y": np.arange(4), "x": np.arange(4)}, + ).chunk({"y": 2, "x": 2}) + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=16, + ) + ds.to_zarr(store=store, mode="w") + root_cid = await store.flush() + + read_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + primary = read_store._primary_array_path + assert primary is not None + + # "c" appears directly under the primary... + top = {key async for key in read_store.list_dir(primary)} + assert "c" in top + + # ...and listing one level deeper enumerates the first chunk coordinate. + chunk_prefix = "c" if primary == "" else f"{primary}/c" + descended = {key async for key in read_store.list_dir(chunk_prefix)} + + expected = { + key[len(chunk_prefix) + 1 :].split("/", 1)[0] + async for key in read_store.list() + if key.startswith(f"{chunk_prefix}/") + } + assert expected, "test setup produced no chunk keys under the primary" + assert descended == expected diff --git a/tests/test_z1_multivar_index.py b/tests/test_z1_multivar_index.py new file mode 100644 index 0000000..3ba5752 --- /dev/null +++ b/tests/test_z1_multivar_index.py @@ -0,0 +1,211 @@ +import base64 +import json +from pathlib import Path + +import dag_cbor +import numpy as np +import pytest +import xarray as xr +from testing_utils import CIDInMemoryCAS + +from py_hamt import HAMT, ShardedZarrStore +from py_hamt.hamt_to_sharded_converter import convert_hamt_to_sharded +from py_hamt.sharded_zarr_store import SHARDED_ZARR_V2 +from py_hamt.zarr_hamt_store import ZarrHAMTStore + +FIXTURE_DIR = Path(__file__).parent / "fixtures" +DATA_VARIABLES = ("temp", "precip") +ARRAY_SHAPE = (4, 4, 6) + + +def _multivar_dataset(*, different_chunk_grids: bool) -> xr.Dataset: + dataset = xr.Dataset( + { + "temp": ( + ("time", "lat", "lon"), + np.full(ARRAY_SHAPE, 1.0, dtype=np.float64), + ), + "precip": ( + ("time", "lat", "lon"), + np.full(ARRAY_SHAPE, 2.0, dtype=np.float64), + ), + }, + coords={ + "time": np.arange(ARRAY_SHAPE[0]), + "lat": np.linspace(-90.0, 90.0, ARRAY_SHAPE[1]), + "lon": np.linspace(-180.0, 180.0, ARRAY_SHAPE[2]), + }, + ) + dataset["temp"].encoding["chunks"] = (2, 4, 6) + dataset["precip"].encoding["chunks"] = ( + (2, 2, 6) if different_chunk_grids else (2, 4, 6) + ) + return dataset + + +def _single_var_dataset() -> xr.Dataset: + values = np.arange(np.prod(ARRAY_SHAPE), dtype=np.float64).reshape(ARRAY_SHAPE) + dataset = xr.Dataset( + {"temp": (("time", "lat", "lon"), values)}, + coords={ + "time": np.arange(ARRAY_SHAPE[0]), + "lat": np.linspace(-90.0, 90.0, ARRAY_SHAPE[1]), + "lon": np.linspace(-180.0, 180.0, ARRAY_SHAPE[2]), + }, + ) + dataset["temp"].encoding["chunks"] = (2, 4, 6) + return dataset + + +async def _decode_root(cas: CIDInMemoryCAS, root_cid: str) -> dict[str, object]: + root = dag_cbor.decode(await cas.load(root_cid)) + assert isinstance(root, dict) + return root + + +def _assert_data_chunks_are_indexed( + root_obj: dict[str, object], array_paths: tuple[str, ...] = DATA_VARIABLES +) -> None: + metadata = root_obj["metadata"] + assert isinstance(metadata, dict) + + for array_path in array_paths: + chunk_prefix = f"{array_path}/c/" + offending_keys = sorted( + key + for key in metadata + if isinstance(key, str) and key.startswith(chunk_prefix) + ) + assert not offending_keys, ( + f"data-variable chunks for {array_path!r} must use a per-array " + f"shard index, not root metadata entries; offending keys: {offending_keys}" + ) + + data_variable_keys = sorted( + key + for key in metadata + if isinstance(key, str) + and any(key == path or key.startswith(f"{path}/") for path in array_paths) + ) + non_metadata_keys = [ + key for key in data_variable_keys if not key.endswith("zarr.json") + ] + assert not non_metadata_keys, ( + "root metadata may contain only zarr.json entries for data variables; " + f"offending keys: {non_metadata_keys}" + ) + + +async def _open_sharded_dataset( + cas: CIDInMemoryCAS, root_cid: str, *, group: str | None = None +) -> xr.Dataset: + store = await ShardedZarrStore.open(cas=cas, read_only=True, root_cid=root_cid) + return xr.open_zarr(store=store, group=group) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("different_chunk_grids", [False, True]) +async def test_multivar_chunks_have_per_array_indexes( + different_chunk_grids: bool, +) -> None: + # Path-aware indexing is the v2 contract; shape-based creation intentionally + # remains the deprecated v1 compatibility path on the current base. + # Distinct per-array chunk grids exercise the core multi-array corruption + # scenario: reuse of the primary array's geometry would corrupt "precip". + expected = _multivar_dataset(different_chunk_grids=different_chunk_grids) + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + chunks_per_shard=8, + manifest_version=SHARDED_ZARR_V2, + ) + + expected.to_zarr(store=store, mode="w", group="0") + root_cid = await store.flush() + + actual = await _open_sharded_dataset(cas, root_cid, group="0") + np.testing.assert_array_equal(actual["temp"].values, expected["temp"].values) + np.testing.assert_array_equal(actual["precip"].values, expected["precip"].values) + _assert_data_chunks_are_indexed( + await _decode_root(cas, root_cid), ("0/temp", "0/precip") + ) + + +@pytest.mark.asyncio +async def test_converter_multivar_builds_per_array_indexes() -> None: + expected = _multivar_dataset(different_chunk_grids=True) + cas = CIDInMemoryCAS() + hamt = await HAMT.build(cas=cas, values_are_bytes=True) + source_store = ZarrHAMTStore(hamt, read_only=False) + + expected.to_zarr(store=source_store, mode="w") + await hamt.make_read_only() + hamt_root_cid = str(hamt.root_node_id) + + # The converter's type hint is narrower than what it needs (KuboCAS vs any + # ContentAddressedStore); any CID-producing CAS works at runtime. + sharded_root_cid = await convert_hamt_to_sharded( + cas=cas, # type: ignore[arg-type] + hamt_root_cid=hamt_root_cid, + chunks_per_shard=8, + ) + + actual = await _open_sharded_dataset(cas, sharded_root_cid) + np.testing.assert_array_equal(actual["temp"].values, expected["temp"].values) + np.testing.assert_array_equal(actual["precip"].values, expected["precip"].values) + _assert_data_chunks_are_indexed(await _decode_root(cas, sharded_root_cid)) + + +@pytest.mark.asyncio +async def test_root_manifest_has_explicit_version_field() -> None: + expected = _single_var_dataset() + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=ARRAY_SHAPE, + chunk_shape=(2, 4, 6), + chunks_per_shard=8, + ) + + expected.to_zarr(store=store, mode="w") + root_obj = await _decode_root(cas, await store.flush()) + + assert "manifest_version" in root_obj + assert isinstance(root_obj["manifest_version"], str) + assert root_obj["manifest_version"] + + +async def _load_fixture(filename: str) -> tuple[CIDInMemoryCAS, str]: + serialized = json.loads((FIXTURE_DIR / filename).read_text()) + cas = CIDInMemoryCAS() + cas.store.update({ + cid: base64.b64decode(encoded_block) + for cid, encoded_block in serialized["blocks"].items() + }) + return cas, serialized["root_cid"] + + +@pytest.mark.asyncio +async def test_legacy_single_var_fixture_readable() -> None: + # Generated at HEAD 790f5a4 by z1_make_legacy_fixture.py. Never regenerate this + # fixture after the format change: it is the legacy compatibility contract. + cas, root_cid = await _load_fixture("z1_legacy_single_var_store.json") + + actual = await _open_sharded_dataset(cas, root_cid) + np.testing.assert_array_equal( + actual["temp"].values, _single_var_dataset()["temp"].values + ) + + +@pytest.mark.asyncio +async def test_legacy_multivar_fixture_readable() -> None: + # Generated at HEAD 790f5a4 by z1_make_legacy_fixture.py. It preserves the + # current mixed shard-index/flat-metadata multi-variable layout for reading. + cas, root_cid = await _load_fixture("z1_legacy_multivar_store.json") + + actual = await _open_sharded_dataset(cas, root_cid) + expected = _multivar_dataset(different_chunk_grids=False) + np.testing.assert_array_equal(actual["temp"].values, expected["temp"].values) + np.testing.assert_array_equal(actual["precip"].values, expected["precip"].values) diff --git a/tests/test_z2_resize_remap.py b/tests/test_z2_resize_remap.py new file mode 100644 index 0000000..b365ffe --- /dev/null +++ b/tests/test_z2_resize_remap.py @@ -0,0 +1,142 @@ +import json + +import numpy as np +import pandas as pd +import pytest +import xarray as xr +import zarr.core.buffer +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore + + +@pytest.mark.asyncio +async def test_append_along_non_first_dimension_preserves_existing_chunks() -> None: + """Changing a trailing chunk-grid dimension must remap existing CIDs.""" + times = pd.date_range("2024-01-01", periods=2) + initial = xr.Dataset( + {"temp": (["time", "lon"], np.arange(8.0).reshape(2, 4))}, + coords={"time": times, "lon": np.arange(4.0)}, + ).chunk({"time": 1, "lon": 2}) + appended = xr.Dataset( + {"temp": (["time", "lon"], 100 + np.arange(4.0).reshape(2, 2))}, + coords={"time": times, "lon": np.arange(4.0, 6.0)}, + ).chunk({"time": 1, "lon": 2}) + + cas = CIDInMemoryCAS() + write_store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(2, 4), + chunk_shape=(1, 2), + chunks_per_shard=16, + ) + initial.to_zarr(store=write_store, mode="w") + appended.to_zarr(store=write_store, append_dim="lon") + root_cid = await write_store.flush() + + read_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + actual = xr.open_zarr(store=read_store).compute() + expected = xr.concat([initial, appended], dim="lon").compute() + + np.testing.assert_array_equal( + actual["temp"].values, + expected["temp"].values, + err_msg="append along non-first dimension scrambled existing chunks", + ) + + +@pytest.mark.asyncio +async def test_secondary_array_metadata_does_not_resize_primary_geometry() -> None: + """Only the primary array's zarr.json may resize the sharded geometry.""" + initial = xr.Dataset( + {"temp": (["time", "lon"], np.arange(8.0).reshape(2, 4))}, + coords={"time": np.arange(2), "lon": np.arange(4)}, + ).chunk({"time": 1, "lon": 2}) + + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(2, 4), + chunk_shape=(1, 2), + chunks_per_shard=16, + ) + initial.to_zarr(store=store, mode="w") + + prototype = zarr.core.buffer.default_buffer_prototype() + secondary_metadata = { + "zarr_format": 3, + "node_type": "array", + "shape": [2, 6], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [1, 2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0.0, + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "attributes": {"_ARRAY_DIMENSIONS": ["time", "lon"]}, + "dimension_names": ["time", "lon"], + } + await store.set( + "other/zarr.json", + prototype.buffer.from_bytes(json.dumps(secondary_metadata).encode()), + ) + + assert store._array_shape == (2, 4), ( + "secondary array metadata spuriously resized the primary geometry" + ) + root_cid = await store.flush() + read_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + actual = xr.open_zarr(store=read_store)["temp"].compute() + xr.testing.assert_identical(initial["temp"].compute(), actual) + + +@pytest.mark.asyncio +async def test_shrink_non_first_dimension_preserves_surviving_chunks() -> None: + """Shrinking a trailing dimension must retain chunks at surviving coordinates.""" + initial = xr.Dataset( + {"temp": (["time", "lon"], np.arange(12.0).reshape(2, 6))}, + coords={"time": np.arange(2), "lon": np.arange(6)}, + ).chunk({"time": 1, "lon": 2}) + + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(2, 6), + chunk_shape=(1, 2), + chunks_per_shard=16, + ) + initial.to_zarr(store=store, mode="w") + await store.resize_store((2, 4)) + await store.resize_variable("temp", (2, 4)) + await store.resize_variable("lon", (4,)) + root_cid = await store.flush() + + read_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + actual = xr.open_zarr(store=read_store, consolidated=False)["temp"].compute() + expected = initial["temp"].isel(lon=slice(0, 4)).compute() + + np.testing.assert_array_equal( + actual.values, + expected.values, + err_msg="shrinking a non-first dimension scrambled surviving chunks", + ) diff --git a/tests/test_z3_shard_cache_eviction.py b/tests/test_z3_shard_cache_eviction.py new file mode 100644 index 0000000..a80316a --- /dev/null +++ b/tests/test_z3_shard_cache_eviction.py @@ -0,0 +1,109 @@ +"""Regression tests for shard eviction while a shard is in active use.""" + +import pytest +import zarr.core.buffer +from dag_cbor.ipld import IPLDKind +from multiformats import CID, multihash + +from py_hamt import ContentAddressedStore, ShardedZarrStore + + +def _normalize_cid(identifier: IPLDKind) -> str: + """Normalize a CID object or string to a stable base32 key.""" + cid = CID.decode(identifier) if isinstance(identifier, str) else identifier + if not isinstance(cid, CID): + raise TypeError( + f"Expected a CID or CID string, got {type(identifier).__name__}" + ) + return cid.set(base="base32").encode("base32") + + +class CIDInMemoryCAS(ContentAddressedStore): + """Fully offline CAS whose ``save`` method returns real CIDs.""" + + def __init__(self) -> None: + self.store: dict[str, bytes] = {} + self._hash_algorithm = multihash.get("blake3") + + async def save(self, data: bytes, codec: ContentAddressedStore.CodecInput) -> CID: + digest = self._hash_algorithm.digest(data, size=32) + cid = CID("base32", 1, codec, digest) + self.store[_normalize_cid(cid)] = data + return cid + + async def load( + self, + identifier: IPLDKind, + offset: int | None = None, + length: int | None = None, + suffix: int | None = None, + ) -> bytes: + data = self.store[_normalize_cid(identifier)] + if offset is not None: + if length is not None: + return data[offset : offset + length] + return data[offset:] + if suffix is not None: + return data[-suffix:] + return data + + +@pytest.mark.asyncio +async def test_read_never_raises_under_cache_pressure() -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(8,), + chunk_shape=(1,), + chunks_per_shard=1, + max_cache_memory_bytes=1, + ) + pointer = str(await cas.save(b"payload", codec="raw")) + prototype = zarr.core.buffer.default_buffer_prototype() + + await store.set_pointer("temp/c/0", pointer) + await store.set_pointer("temp/c/1", pointer) + assert await store.get("temp/c/2", prototype) is None + assert await store.get("temp/c/2", prototype) is None + + chunk_zero = await store.get("temp/c/0", prototype) + assert chunk_zero is not None + assert chunk_zero.to_bytes() == b"payload" + assert await store.get("temp/c/3", prototype) is None + + +@pytest.mark.asyncio +async def test_no_lost_acknowledged_writes_under_cache_pressure() -> None: + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + array_shape=(16,), + chunk_shape=(1,), + chunks_per_shard=1, + max_cache_memory_bytes=1, + ) + prototype = zarr.core.buffer.default_buffer_prototype() + expected_payloads = [f"payload-{index}".encode() for index in range(16)] + + for index, payload in enumerate(expected_payloads): + pointer = str(await cas.save(payload, codec="raw")) + key = f"temp/c/{index}" + await store.set_pointer(key, pointer) + written_chunk = await store.get(key, prototype) + assert written_chunk is not None + assert written_chunk.to_bytes() == payload + + root_cid = await store.flush() + persisted_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + max_cache_memory_bytes=1, + ) + + for index, expected_payload in enumerate(expected_payloads): + persisted_chunk = await persisted_store.get(f"temp/c/{index}", prototype) + assert persisted_chunk is not None + assert persisted_chunk.to_bytes() == expected_payload diff --git a/tests/test_z4_coord_allowlist.py b/tests/test_z4_coord_allowlist.py new file mode 100644 index 0000000..71bdd97 --- /dev/null +++ b/tests/test_z4_coord_allowlist.py @@ -0,0 +1,46 @@ +import numpy as np +import pandas as pd +import pytest +import xarray as xr +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore +from py_hamt.sharded_zarr_store import SHARDED_ZARR_V2 + + +@pytest.mark.asyncio +async def test_non_allowlisted_coordinate_names_round_trip() -> None: + """Coordinate arrays must not be indexed using the primary array geometry.""" + times = pd.date_range("2024-01-01", periods=4) + dataset = xr.Dataset( + { + "temp": ( + ["time", "y", "x"], + np.arange(4 * 4 * 6, dtype=np.float64).reshape(4, 4, 6), + ) + }, + coords={ + "time": times, + "y": np.arange(4, dtype=np.float64), + "x": np.arange(6, dtype=np.float64), + }, + ).chunk({"time": 2, "y": 4, "x": 6}) + + cas = CIDInMemoryCAS() + write_store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + chunks_per_shard=8, + manifest_version=SHARDED_ZARR_V2, + ) + dataset.to_zarr(store=write_store, mode="w", group="0") + root_cid = await write_store.flush() + + read_store = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=root_cid, + ) + actual = xr.open_zarr(store=read_store, group="0") + + xr.testing.assert_identical(dataset.compute(), actual.compute()) diff --git a/tests/test_z5_delete_stale_metadata_cache.py b/tests/test_z5_delete_stale_metadata_cache.py new file mode 100644 index 0000000..02025b4 --- /dev/null +++ b/tests/test_z5_delete_stale_metadata_cache.py @@ -0,0 +1,53 @@ +import pytest +import zarr.core.buffer + +from py_hamt import ( + HAMT, + InMemoryCAS, + SimpleEncryptedZarrHAMTStore, + ZarrHAMTStore, +) + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +METADATA_KEY = "temp/zarr.json" +METADATA_PAYLOAD = b'{"shape": [10]}' + + +async def _assert_deleted_metadata_is_not_cached(store: ZarrHAMTStore) -> None: + await store.set(METADATA_KEY, PROTOTYPE.buffer.from_bytes(METADATA_PAYLOAD)) + + cached_value = await store.get(METADATA_KEY, PROTOTYPE) + assert cached_value is not None + assert cached_value.to_bytes() == METADATA_PAYLOAD + + await store.delete(METADATA_KEY) + + assert not await store.exists(METADATA_KEY) + assert METADATA_KEY not in [key async for key in store.list()] + + deleted_value = await store.get(METADATA_KEY, PROTOTYPE) + stale_value = None if deleted_value is None else deleted_value.to_bytes() + assert deleted_value is None, ( + f"get() returned stale metadata after delete: {stale_value!r}" + ) + + +@pytest.mark.asyncio +async def test_plain_delete_purges_metadata_read_cache() -> None: + hamt = await HAMT.build(cas=InMemoryCAS(), values_are_bytes=True) + store = ZarrHAMTStore(hamt, read_only=False) + + await _assert_deleted_metadata_is_not_cached(store) + + +@pytest.mark.asyncio +async def test_encrypted_delete_purges_metadata_read_cache() -> None: + hamt = await HAMT.build(cas=InMemoryCAS(), values_are_bytes=True) + store = SimpleEncryptedZarrHAMTStore( + hamt, + read_only=False, + encryption_key=b"\x01" * 32, + header=b"z5-delete-cache-test", + ) + + await _assert_deleted_metadata_is_not_cached(store) diff --git a/tests/test_z6_store_contract.py b/tests/test_z6_store_contract.py new file mode 100644 index 0000000..889ff63 --- /dev/null +++ b/tests/test_z6_store_contract.py @@ -0,0 +1,140 @@ +import json + +import pytest +import zarr +from testing_utils import CIDInMemoryCAS +from zarr.abc.store import ( + OffsetByteRequest, + RangeByteRequest, + SuffixByteRequest, +) + +from py_hamt import ShardedZarrStore + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4, 4], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2, 2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "attributes": {}, + }, + separators=(",", ":"), +).encode() + + +def buf(data: bytes) -> zarr.core.buffer.Buffer: + return PROTOTYPE.buffer.from_bytes(data) + + +async def populated_store() -> ShardedZarrStore: + store = await ShardedZarrStore.open( + cas=CIDInMemoryCAS(), + read_only=False, + array_shape=(4, 4), + chunk_shape=(2, 2), + chunks_per_shard=2, + ) + await store.set("temp/zarr.json", buf(ARRAY_METADATA)) + await store.set("temp/c/0/0", buf(b"zero-zero")) + await store.set("temp/c/1/1", buf(b"one-one")) + return store + + +@pytest.mark.asyncio +async def test_get_metadata_byte_range_range() -> None: + store = await populated_store() + + result = await store.get( + "temp/zarr.json", + PROTOTYPE, + byte_range=RangeByteRequest(start=0, end=5), + ) + + assert result is not None + assert result.to_bytes() == ARRAY_METADATA[0:5] + + +@pytest.mark.asyncio +async def test_get_metadata_byte_range_offset() -> None: + store = await populated_store() + + result = await store.get( + "temp/zarr.json", + PROTOTYPE, + byte_range=OffsetByteRequest(offset=5), + ) + + assert result is not None + assert result.to_bytes() == ARRAY_METADATA[5:] + + +@pytest.mark.asyncio +async def test_get_metadata_byte_range_suffix() -> None: + store = await populated_store() + + result = await store.get( + "temp/zarr.json", + PROTOTYPE, + byte_range=SuffixByteRequest(suffix=5), + ) + + assert result is not None + assert result.to_bytes() == ARRAY_METADATA[-5:] + + +@pytest.mark.asyncio +async def test_list_yields_chunk_keys() -> None: + store = await populated_store() + + keys = {key async for key in store.list()} + + assert keys == {"temp/zarr.json", "temp/c/0/0", "temp/c/1/1"} + + +@pytest.mark.asyncio +async def test_list_prefix_yields_chunk_keys() -> None: + store = await populated_store() + + keys = {key async for key in store.list_prefix("temp/c/")} + + assert keys == {"temp/c/0/0", "temp/c/1/1"} + + +@pytest.mark.asyncio +async def test_list_dir_named_prefix() -> None: + store = await populated_store() + + children = {child async for child in store.list_dir("temp")} + + assert children == {"zarr.json", "c"} + + +@pytest.mark.asyncio +async def test_root_level_chunk_key_no_silent_clobber() -> None: + store = await populated_store() + + try: + await store.set("c/0/0", buf(b"ROOT-CHUNK")) + except ValueError: + # Rejecting root-level chunks explicitly is an acceptable contract + # (_validate_chunk_write_key raises ValueError for this case). + pass + else: + named = await store.get("temp/c/0/0", PROTOTYPE) + assert named is not None and named.to_bytes() == b"zero-zero", ( + "root-level chunk write silently clobbered named array chunk" + ) + rooted = await store.get("c/0/0", PROTOTYPE) + assert rooted is not None and rooted.to_bytes() == b"ROOT-CHUNK" diff --git a/tests/test_z7_delete_legacy_metadata.py b/tests/test_z7_delete_legacy_metadata.py new file mode 100644 index 0000000..d67a799 --- /dev/null +++ b/tests/test_z7_delete_legacy_metadata.py @@ -0,0 +1,247 @@ +import json +from typing import Literal, TypedDict, cast + +import dag_cbor +import pytest +import zarr +from dag_cbor.ipld import IPLDKind +from hypothesis import given, settings +from hypothesis import strategies as st +from multiformats import CID +from testing_utils import CIDInMemoryCAS + +from py_hamt import ShardedZarrStore + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +CHUNK_KEY = "myarr/c/0/0" +LegacyAction = tuple[Literal["set", "delete", "flush_reopen"], bytes] +ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4, 4], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2, 2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes"}], + }, + separators=(",", ":"), +).encode() + + +class _ArrayManifest(TypedDict): + shard_cids: list[CID | None] + + +class _IPLDV2Root(TypedDict): + metadata: dict[str, IPLDKind] + arrays: dict[str, _ArrayManifest] + + +def buf(data: bytes) -> zarr.core.buffer.Buffer: + return PROTOTYPE.buffer.from_bytes(data) + + +async def _base_store() -> tuple[CIDInMemoryCAS, str]: + """A flushed v2 store with one registered array and one shard chunk.""" + cas = CIDInMemoryCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + chunks_per_shard=4, + ) + await store.set("myarr/zarr.json", buf(ARRAY_METADATA)) + await store.set(CHUNK_KEY, buf(b"current")) + root_cid = await store.flush() + return cas, root_cid + + +async def _decoded_root( + cas: CIDInMemoryCAS, + root_cid: str, +) -> _IPLDV2Root: + root_obj = dag_cbor.decode(await cas.load(root_cid)) + assert isinstance(root_obj, dict) + return cast(_IPLDV2Root, root_obj) + + +async def dual_representation_store() -> tuple[CIDInMemoryCAS, ShardedZarrStore]: + """A writable store whose chunk key exists BOTH as a populated shard slot + and as a stale legacy entry in root metadata, mimicking a legacy root that + was later overwritten by an older py-hamt without metadata cleanup.""" + cas, root_cid = await _base_store() + + root_obj = await _decoded_root(cas, root_cid) + stale_cid = await cas.save(b"stale-legacy", codec="raw") + metadata = root_obj["metadata"] + assert isinstance(metadata, dict) + metadata[CHUNK_KEY] = stale_cid + legacy_root_cid = await cas.save( + dag_cbor.encode(cast(IPLDKind, root_obj)), + codec="dag-cbor", + ) + + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=str(legacy_root_cid), + ) + # Preconditions: both representations exist, and the shard slot wins reads. + assert CHUNK_KEY in store._root_obj["metadata"] + current = await store.get(CHUNK_KEY, PROTOTYPE) + assert current is not None + assert current.to_bytes() == b"current" + return cas, store + + +async def shard_empty_legacy_store() -> tuple[CIDInMemoryCAS, ShardedZarrStore]: + """A writable store whose chunk key exists ONLY as a legacy metadata entry + (shard slot empty), the layout served by the legacy fallback.""" + cas, root_cid = await _base_store() + + root_obj = await _decoded_root(cas, root_cid) + arrays = root_obj["arrays"] + assert isinstance(arrays, dict) + shard_cids = arrays["myarr"]["shard_cids"] + shard_cid = shard_cids[0] + assert isinstance(shard_cid, CID) + shard_entries = dag_cbor.decode(await cas.load(shard_cid)) + assert isinstance(shard_entries, list) + original_chunk_cid = shard_entries[0] + assert isinstance(original_chunk_cid, CID) + + shard_cids[0] = None + metadata = root_obj["metadata"] + assert isinstance(metadata, dict) + metadata[CHUNK_KEY] = original_chunk_cid + legacy_root_cid = await cas.save( + dag_cbor.encode(cast(IPLDKind, root_obj)), + codec="dag-cbor", + ) + + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=str(legacy_root_cid), + ) + original = await store.get(CHUNK_KEY, PROTOTYPE) + assert original is not None + assert original.to_bytes() == b"current" + return cas, store + + +async def assert_optional_value( + store: ShardedZarrStore, + expected: bytes | None, +) -> None: + value = await store.get(CHUNK_KEY, PROTOTYPE) + if expected is None: + assert value is None, value.to_bytes() if value is not None else None + else: + assert value is not None + assert value.to_bytes() == expected + + +@pytest.mark.asyncio +async def test_overwrite_removes_legacy_metadata_entry() -> None: + cas, store = await shard_empty_legacy_store() + + await store.set(CHUNK_KEY, buf(b"overwritten")) + + new_root_cid = await store.flush() + root_obj = await _decoded_root(cas, new_root_cid) + metadata = root_obj["metadata"] + assert isinstance(metadata, dict) + assert CHUNK_KEY not in metadata, ( + "overwriting a legacy chunk must not pin the superseded CID in metadata" + ) + overwritten = await store.get(CHUNK_KEY, PROTOTYPE) + assert overwritten is not None + assert overwritten.to_bytes() == b"overwritten" + + +@pytest.mark.asyncio +async def test_delete_removes_legacy_metadata_when_shard_slot_exists() -> None: + _, store = await dual_representation_store() + + await store.delete(CHUNK_KEY) + + deleted = await store.get(CHUNK_KEY, PROTOTYPE) + assert deleted is None, deleted.to_bytes() if deleted is not None else None + + +@pytest.mark.asyncio +async def test_delete_persists_legacy_metadata_removal_after_reopen() -> None: + cas, store = await dual_representation_store() + + await store.delete(CHUNK_KEY) + new_root_cid = await store.flush() + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=new_root_cid, + ) + + deleted = await reopened.get(CHUNK_KEY, PROTOTYPE) + assert deleted is None, deleted.to_bytes() if deleted is not None else None + + +@pytest.mark.asyncio +@given( + actions=st.lists( + st.one_of( + st.tuples(st.just("set"), st.binary(max_size=64)), + st.tuples(st.just("delete"), st.just(b"")), + st.tuples(st.just("flush_reopen"), st.just(b"")), + ), + min_size=1, + max_size=10, + ), +) +@settings(max_examples=25, deadline=None) +async def test_legacy_chunk_mutation_sequence_property( + actions: list[LegacyAction], +) -> None: + """No operation history may revive the superseded legacy metadata CID.""" + cas, store = await shard_empty_legacy_store() + expected: bytes | None = b"current" + mutation_seen = False + + for action, payload in actions: + if action == "set": + await store.set(CHUNK_KEY, buf(payload)) + expected = payload + mutation_seen = True + elif action == "delete": + await store.delete(CHUNK_KEY) + expected = None + mutation_seen = True + else: + checkpoint_root = await store.flush() + checkpoint_obj = await _decoded_root(cas, checkpoint_root) + if mutation_seen: + assert CHUNK_KEY not in checkpoint_obj["metadata"] + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + root_cid=checkpoint_root, + ) + await assert_optional_value(store, expected) + + final_root = await store.flush() + final_root_obj = await _decoded_root(cas, final_root) + if mutation_seen: + assert CHUNK_KEY not in final_root_obj["metadata"] + reopened = await ShardedZarrStore.open( + cas=cas, + read_only=True, + root_cid=final_root, + ) + await assert_optional_value(reopened, expected) diff --git a/tests/test_z8_cas_save_cid_validation.py b/tests/test_z8_cas_save_cid_validation.py new file mode 100644 index 0000000..99d9eee --- /dev/null +++ b/tests/test_z8_cas_save_cid_validation.py @@ -0,0 +1,62 @@ +import json + +import pytest +import zarr +from dag_cbor.ipld import IPLDKind +from testing_utils import CIDInMemoryCAS + +from py_hamt import ContentAddressedStore, ShardedZarrStore +from py_hamt.sharded_zarr_store import SHARDED_ZARR_V2 + +PROTOTYPE = zarr.core.buffer.default_buffer_prototype() +ARRAY_METADATA = json.dumps( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4, 4], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": [2, 2]}, + }, + "chunk_key_encoding": { + "name": "default", + "configuration": {"separator": "/"}, + }, + "fill_value": 0, + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "attributes": {}, + }, + separators=(",", ":"), +).encode() + + +class ToggleRawBytesCAS(CIDInMemoryCAS): + def __init__(self) -> None: + super().__init__() + self.return_raw_bytes = False + + async def save( + self, data: bytes, codec: ContentAddressedStore.CodecInput + ) -> IPLDKind: # type: ignore[override] + cid = await super().save(data, codec) + if self.return_raw_bytes and codec == "raw": + return self._hash_algorithm.digest(data, size=32) + return cid + + +@pytest.mark.asyncio +async def test_set_rejects_non_cid_from_cas_save() -> None: + cas = ToggleRawBytesCAS() + store = await ShardedZarrStore.open( + cas=cas, + read_only=False, + chunks_per_shard=2, + manifest_version=SHARDED_ZARR_V2, + ) + await store.set("temp/zarr.json", PROTOTYPE.buffer.from_bytes(ARRAY_METADATA)) + + cas.return_raw_bytes = True + + with pytest.raises((TypeError, RuntimeError), match="CID"): + await store.set("temp/c/0/0", PROTOTYPE.buffer.from_bytes(b"chunk data")) diff --git a/tests/test_z9_cache_contains.py b/tests/test_z9_cache_contains.py new file mode 100644 index 0000000..609d601 --- /dev/null +++ b/tests/test_z9_cache_contains.py @@ -0,0 +1,23 @@ +import warnings + +import pytest + +from py_hamt.sharded_zarr_store import MemoryBoundedLRUCache + + +@pytest.mark.asyncio +async def test_memory_bounded_lru_cache_contains_uses_in_operator() -> None: + cache = MemoryBoundedLRUCache(max_memory_bytes=1_024) + present_key = ("present", 0) + absent_key = ("some", 0) + + await cache.put(present_key, [None]) + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"coroutine .* was never awaited", + category=RuntimeWarning, + ) + assert (present_key in cache) is True + assert (absent_key in cache) is False diff --git a/tests/test_zarr_ipfs.py b/tests/test_zarr_ipfs.py index a7769b0..69d1db1 100644 --- a/tests/test_zarr_ipfs.py +++ b/tests/test_zarr_ipfs.py @@ -56,7 +56,7 @@ def random_zarr_dataset(): # This test also collects miscellaneous statistics about performance, run with pytest -s to see these statistics being printed out -@pytest.mark.asyncio(loop_scope="session") # ← match the loop of the fixture +@pytest.mark.asyncio async def test_write_read( create_ipfs: tuple[str, str], random_zarr_dataset: xr.Dataset, diff --git a/tests/test_zarr_ipfs_partial.py b/tests/test_zarr_ipfs_partial.py index 84de087..364c965 100644 --- a/tests/test_zarr_ipfs_partial.py +++ b/tests/test_zarr_ipfs_partial.py @@ -58,7 +58,7 @@ def random_zarr_dataset(): # This test also collects miscellaneous statistics about performance, run with pytest -s to see these statistics being printed out -@pytest.mark.asyncio(loop_scope="session") # ← match the loop of the fixture +@pytest.mark.asyncio async def test_write_read( create_ipfs, random_zarr_dataset: xr.Dataset, diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 229762c..2c433b0 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -5,9 +5,55 @@ from urllib.parse import urlparse import pytest +from dag_cbor.ipld import IPLDKind from hypothesis import strategies as st from hypothesis.strategies import SearchStrategy -from multiformats import CID +from multiformats import CID, multihash + +from py_hamt import ContentAddressedStore + + +def _normalize_cid(identifier: IPLDKind) -> str: + """Normalize a CID object or string to a stable base32 key.""" + if isinstance(identifier, CID): + cid = identifier + elif isinstance(identifier, str): + cid = CID.decode(identifier) + else: + raise TypeError( + f"Expected a CID or CID string, got {type(identifier).__name__}" + ) + return cid.set(base="base32").encode("base32") + + +class CIDInMemoryCAS(ContentAddressedStore): + """Offline in-memory CAS whose object identifiers are valid CIDs.""" + + def __init__(self) -> None: + self.store: dict[str, bytes] = {} + self._hash_algorithm = multihash.get("blake3") + + async def save(self, data: bytes, codec: ContentAddressedStore.CodecInput) -> CID: + digest = self._hash_algorithm.digest(data, size=32) + cid = CID("base32", 1, codec, digest) + self.store[_normalize_cid(cid)] = data + return cid + + async def load( + self, + id: IPLDKind, + offset: int | None = None, + length: int | None = None, + suffix: int | None = None, + ) -> bytes: + data = self.store[_normalize_cid(id)] + if offset is not None: + if length is not None: + return data[offset : offset + length] + return data[offset:] + if suffix is not None: + return data[-suffix:] + return data def cid_strategy() -> SearchStrategy: diff --git a/uv.lock b/uv.lock index 2afcec8..c6479d8 100644 --- a/uv.lock +++ b/uv.lock @@ -168,6 +168,91 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/7e/3db2bd1b1f9e95f7cddca6d6e75e2f2bd9f51b1246e546d88addca0106bd/certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3", size = 159618, upload-time = "2025-04-26T02:12:27.662Z" }, ] +[[package]] +name = "cffi" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, + { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, + { url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" }, + { url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" }, + { url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" }, + { url = "https://files.pythonhosted.org/packages/b0/80/c138990aa2a70b1a269f6e06348729836d733d6f970867943f61d367f8cc/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a", size = 175269, upload-time = "2026-07-06T21:32:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/a8/eb/f636456ff21a83fc13c032b58cc5dde061691546ac79efa284b2989b7982/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384", size = 185881, upload-time = "2026-07-06T21:32:59.253Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, + { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, + { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/45c3a76ad8d43ad9261f4c95436da61128d3ca545d72b9612c0ab5be0b1c/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a", size = 184795, upload-time = "2026-07-06T21:33:06.699Z" }, + { url = "https://files.pythonhosted.org/packages/84/4c/82f132cb4418ee6d953d982b19191e87e2a6372c8a4ce36e50b69d6ade4a/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea", size = 184746, upload-time = "2026-07-06T21:33:08.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, + { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, + { url = "https://files.pythonhosted.org/packages/a6/cf/2b684132056f438567b61e19d690dd31cd0921ace051e0a458be6074369e/cffi-2.1.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0", size = 208801, upload-time = "2026-07-06T21:33:13.617Z" }, + { url = "https://files.pythonhosted.org/packages/6f/08/f2e7d62c460faae0926f2d6e423694aa409ced3bc1fe2927a0a6e5f05416/cffi-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224", size = 221808, upload-time = "2026-07-06T21:33:15.466Z" }, + { url = "https://files.pythonhosted.org/packages/38/37/04f54b8e63a02f3d908332c9effbf8c366167c6f733ed8a3d4f79b7e2a1e/cffi-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c", size = 225241, upload-time = "2026-07-06T21:33:16.869Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d6/c72eecca433cd3e681c65ed313ab4835d9d4a379704d0f628a6a05f51c2e/cffi-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a", size = 223588, upload-time = "2026-07-06T21:33:18.239Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4b/e706f67279140f92939da3475ad610df18bfd52d50f14953a8e5fede71d5/cffi-2.1.0-cp313-cp313-win32.whl", hash = "sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2", size = 175248, upload-time = "2026-07-06T21:33:19.799Z" }, + { url = "https://files.pythonhosted.org/packages/5a/47/59eb7975cb0e4ef0afa764ea945b29a5bb4537a9f771cb7d6c8a5dd74c95/cffi-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512", size = 185717, upload-time = "2026-07-06T21:33:21.47Z" }, + { url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, + { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/20/71/7c8372d30e42415602ed9f268f7cfd66f1b855fed881ecd168bcb45dbc0b/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d", size = 184965, upload-time = "2026-07-06T21:33:26.605Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/584e626835f0375c928176c04137c96927165cb8733cdb3150ec04e5ee5e/cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac", size = 184952, upload-time = "2026-07-06T21:33:27.823Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, + { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d7/97d3136f81db489ec8d1d67748c110d6c994268fd7528014aa9f2b085e4e/cffi-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5", size = 221593, upload-time = "2026-07-06T21:33:33.044Z" }, + { url = "https://files.pythonhosted.org/packages/d3/27/93195977168ee63aed233a1a0993a2178798654d1f4bddcdd321d6fd3b21/cffi-2.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce", size = 225146, upload-time = "2026-07-06T21:33:34.224Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c1/6dbd291ee2ae5a50a034aa057207081f545923bbf15dad4511e985aafff5/cffi-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326", size = 223240, upload-time = "2026-07-06T21:33:35.57Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6f/ade5ce9863a57992a6ea3d0d10d7e29b8749fc127204b3d493d667b2815f/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd", size = 177723, upload-time = "2026-07-06T21:33:51.626Z" }, + { url = "https://files.pythonhosted.org/packages/41/de/92b9eeed4ae4a21d6fd9b2a2c8505cbed573299902ea73981cc13f7ff62c/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", size = 187937, upload-time = "2026-07-06T21:33:53.403Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/cc6ae6c2913a03aab8898eee57963cf1035b8df5872ed8b9115fcc7e2be8/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804", size = 183001, upload-time = "2026-07-06T21:33:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/14/f0/134c00ce0779ec86dea2aa1aac69339c2741a8045072676763512363a2ea/cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714", size = 188538, upload-time = "2026-07-06T21:33:36.792Z" }, + { url = "https://files.pythonhosted.org/packages/50/d8/3b86aba791cb610d24e8a3e1b2cd529e71fa15096b04e4d4e360049d4a4c/cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376", size = 188230, upload-time = "2026-07-06T21:33:38.011Z" }, + { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, + { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, + { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, + { url = "https://files.pythonhosted.org/packages/cc/82/3d5c705acb7abbba9bbd7d79b8e62e0f25b6120eb7ae6ac49f1b721722fe/cffi-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056", size = 223933, upload-time = "2026-07-06T21:33:43.603Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d0/47e338384ab6b1004241002fa616301020cea4fc95f283506565d252f276/cffi-2.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4", size = 226749, upload-time = "2026-07-06T21:33:45.046Z" }, + { url = "https://files.pythonhosted.org/packages/70/25/65bd5b58ea4bfdfc15cde02cb5365f89ef8ab8b2adfb8fe5c4bd4233382f/cffi-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94", size = 225703, upload-time = "2026-07-06T21:33:46.374Z" }, + { url = "https://files.pythonhosted.org/packages/dc/78/aa01ac599a8a4322533d45a1f9bc93b338276d2d59dabbe7c6d92a775c81/cffi-2.1.0-cp314-cp314t-win32.whl", hash = "sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76", size = 182857, upload-time = "2026-07-06T21:33:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/b9/26/d00496b22de4d4228f32dde94ad996f350c8aad676d63bcca0743c8dea4d/cffi-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5", size = 194065, upload-time = "2026-07-06T21:33:48.953Z" }, + { url = "https://files.pythonhosted.org/packages/d5/dd/0c7dbf815a579ff005008a2d815a55d6bb047c349eef536d9dc53d3f0a8d/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8", size = 186404, upload-time = "2026-07-06T21:33:50.309Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/ed/da/4bbe583a3b3a5c8c60892124fe17f3fa3656523faf0d3484eae90f091853/cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3", size = 184936, upload-time = "2026-07-06T21:33:58.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4b/1f4c36ab273980d7aa75bb126ea4f8971f24a96108acad3a0a084028c57b/cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc", size = 185045, upload-time = "2026-07-06T21:34:00.085Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3f/0b04a700dd64f465c93020253a793a82c9b4dff9961f48facd0df945d9b8/cffi-2.1.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1", size = 221649, upload-time = "2026-07-06T21:34:06.157Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/b7379a5704c79eda57ce075869ba70a0368d1c850f803b3c0d078d39dcaf/cffi-2.1.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28", size = 225203, upload-time = "2026-07-06T21:34:07.489Z" }, + { url = "https://files.pythonhosted.org/packages/5a/02/d5e6c43ea85c41bda2a184a3418f195fe7cf602967a8d2b94e085b83deef/cffi-2.1.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629", size = 223263, upload-time = "2026-07-06T21:34:08.712Z" }, + { url = "https://files.pythonhosted.org/packages/2c/d8/772b8259bf75749adffb1c546828978381fb516f60cf701f6c83daf60c85/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6", size = 177696, upload-time = "2026-07-06T21:34:26.355Z" }, + { url = "https://files.pythonhosted.org/packages/2f/dd/afa2191fc6d57fedd26e5844a2fe2fcc0bbfa00961bbaa5a41e4921e7cca/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853", size = 187914, upload-time = "2026-07-06T21:34:27.58Z" }, + { url = "https://files.pythonhosted.org/packages/05/ef/6cd4f8c671517162379dc79cfae5aea9106bc38abb89628d5c16adf6a838/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda", size = 183004, upload-time = "2026-07-06T21:34:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/11/b6/12fc55092817a5faa26fb8c40c7f9d662e11a46ee248c137aafc42517d92/cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc", size = 188378, upload-time = "2026-07-06T21:34:09.926Z" }, + { url = "https://files.pythonhosted.org/packages/8d/2e/cdac88979f295fde5daa69622c7d2111e56e7ceb94f211357fbe452339e4/cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca", size = 188319, upload-time = "2026-07-06T21:34:11.101Z" }, + { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, + { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, + { url = "https://files.pythonhosted.org/packages/19/e5/d3cc82a4a0be7902af279c04181ad038449c096734464a5ae1de3e1401bd/cffi-2.1.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f", size = 223843, upload-time = "2026-07-06T21:34:17.509Z" }, + { url = "https://files.pythonhosted.org/packages/b9/65/b434abc97ce7cecc2c640fde160507c0ecc7e21544b483ba3325d2e2ea17/cffi-2.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc", size = 226773, upload-time = "2026-07-06T21:34:19.05Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9f/d4dc66ca651eb1145a133314cda721abf13cfac3d28c4a0402263ae6ad75/cffi-2.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9", size = 225719, upload-time = "2026-07-06T21:34:20.576Z" }, + { url = "https://files.pythonhosted.org/packages/68/5a/e536c528bc8057496c360c0978559a2dc45653f89dd6151078aa7d8fca1a/cffi-2.1.0-cp315-cp315t-win32.whl", hash = "sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b", size = 182760, upload-time = "2026-07-06T21:34:22.059Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/0ffe8b82d3875bced5fa1e7986a7a46b748262a40ab7f60b475eb9fb1bb3/cffi-2.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5", size = 193769, upload-time = "2026-07-06T21:34:23.589Z" }, + { url = "https://files.pythonhosted.org/packages/a0/17/1073b53b68c9b5ca6914adf5f8bf55aacc2d3be102418c90700160ea8605/cffi-2.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210", size = 186405, upload-time = "2026-07-06T21:34:24.857Z" }, +] + [[package]] name = "cfgv" version = "3.4.0" @@ -389,6 +474,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/c4/0b3eee04dac195f4730d102d7a9fbea894ae7a32ce075f84336df96a385d/crc32c-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:eee2a43b663feb6c79a6c1c6e5eae339c2b72cfac31ee54ec0209fa736cf7ee5", size = 39781, upload-time = "2024-09-24T06:19:08.182Z" }, ] +[[package]] +name = "cryptography" +version = "49.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, + { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, + { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, + { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, + { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, + { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, + { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, + { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, + { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, + { url = "https://files.pythonhosted.org/packages/ec/9e/db72b3ae7fc9cfad53e630e56c6ae83b9b6ff0bf3718ffb8012d20b3aabf/cryptography-49.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7", size = 4013892, upload-time = "2026-06-12T20:02:10.735Z" }, + { url = "https://files.pythonhosted.org/packages/86/12/c48a424f38db03027be9f7ed5c7dc5de9933dbee992865f98b13727a009d/cryptography-49.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d", size = 4678835, upload-time = "2026-06-12T20:02:48.743Z" }, + { url = "https://files.pythonhosted.org/packages/68/28/8a3ad4653662c93fc44dc4e5d8fd374c25c42e07b34bbfbadf49cf57a5a8/cryptography-49.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa", size = 4697239, upload-time = "2026-06-12T20:02:56.03Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b2/2193fc74f81aee4f9b62733133b73b5176718932ed8f2e4b03fa040480a6/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb", size = 4685593, upload-time = "2026-06-12T20:02:50.666Z" }, + { url = "https://files.pythonhosted.org/packages/47/f1/1d3eaa243bfc5de4a187b22aa8c048b3e4980bfbe830ac46e6bac2e66947/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d", size = 5289961, upload-time = "2026-06-12T20:01:46.468Z" }, + { url = "https://files.pythonhosted.org/packages/58/39/2d51306721330c486495853eda1c567880ff036de15a14c4b74f399934af/cryptography-49.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561", size = 4731145, upload-time = "2026-06-12T20:02:16.832Z" }, + { url = "https://files.pythonhosted.org/packages/17/50/983e838c7fd0d87fd8c969bcdd328edaf5f756e38df5281637424c155873/cryptography-49.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122", size = 4321719, upload-time = "2026-06-12T20:02:52.611Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f5/8f571d7e27c55bce9f76f026143bcb1e040a4233149ecca0bea5fa5dd5f7/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505", size = 4685209, upload-time = "2026-06-12T20:02:07.282Z" }, + { url = "https://files.pythonhosted.org/packages/e7/84/0e27016a6fc5a0886f797018b26aa42f40c09a82332bff77822a451deaaa/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866", size = 5246285, upload-time = "2026-06-12T20:01:32.439Z" }, + { url = "https://files.pythonhosted.org/packages/11/2d/5e1fb307cb5931881516b464c98774b3f2c36b5d4bb9a2830253cf553cad/cryptography-49.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8", size = 4730441, upload-time = "2026-06-12T20:02:01.469Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c0/bff5a02ee731d207d6a1ed51732549d8c53d2bc8da1d10ec6f2844201d68/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3", size = 4815869, upload-time = "2026-06-12T20:01:36.574Z" }, + { url = "https://files.pythonhosted.org/packages/b9/26/814681d14248d95d73d5c3eea0c39a94eb8302df966f670a2c60de90974b/cryptography-49.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27", size = 4960948, upload-time = "2026-06-12T20:02:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/93ecac273d3738939d023612ad12cca9a3740a5345d69fda04134c43fd96/cryptography-49.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61", size = 3799153, upload-time = "2026-06-12T20:01:39.059Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, + { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, + { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, + { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, + { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, + { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -1521,7 +1656,7 @@ wheels = [ [[package]] name = "py-hamt" -version = "3.4.0" +version = "3.4.1" source = { editable = "." } dependencies = [ { name = "dag-cbor" }, @@ -1549,6 +1684,7 @@ dev = [ { name = "pytest-cov" }, { name = "ruff" }, { name = "snakeviz" }, + { name = "trustme" }, { name = "types-docker" }, { name = "xarray", extra = ["complete"] }, ] @@ -1580,6 +1716,7 @@ dev = [ { name = "pytest-cov", specifier = ">=6.0.0" }, { name = "ruff", specifier = ">=0.7.1" }, { name = "snakeviz", specifier = ">=2.2.0" }, + { name = "trustme", specifier = ">=1.2.1" }, { name = "types-docker", specifier = ">=7.1.0.20250523" }, { name = "xarray", extras = ["complete"], specifier = ">=2025.3.0" }, ] @@ -1619,6 +1756,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/40/ad395740cd641869a13bcf60851296c89624662575621968dcfafabaa7f6/pyarrow-20.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:82f1ee5133bd8f49d31be1299dc07f585136679666b502540db854968576faf9", size = 25944982, upload-time = "2025-04-27T12:33:04.72Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pycryptodome" version = "3.23.0" @@ -2083,6 +2229,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/a7/535c44c7bea4578e48281d83c615219f3ab19e6abc67625ef637c73987be/tornado-6.5.1-cp39-abi3-win_arm64.whl", hash = "sha256:02420a0eb7bf617257b9935e2b754d1b63897525d8a289c9d65690d580b4dcf7", size = 443596, upload-time = "2025-05-22T18:15:37.433Z" }, ] +[[package]] +name = "trustme" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/c5/931476f4cf1cd9e736f32651005078061a50dc164a2569fb874e00eb2786/trustme-1.2.1.tar.gz", hash = "sha256:6528ba2bbc7f2db41f33825c8dd13e3e3eb9d334ba0f909713c8c3139f4ae47f", size = 26844, upload-time = "2025-01-02T01:55:32.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/f3/c34dbabf6da5eda56fe923226769d40e11806952cd7f46655dd06e10f018/trustme-1.2.1-py3-none-any.whl", hash = "sha256:d768e5fc57c86dfc5ec9365102e9b092541cd6954b35d8c1eea01a84f35a762a", size = 16530, upload-time = "2025-01-02T01:55:30.181Z" }, +] + [[package]] name = "types-docker" version = "7.1.0.20250523"