From 5a05fb5aaed01872e8dd5d0d9286287aba025d2c Mon Sep 17 00:00:00 2001 From: Jeff Rhoades Date: Thu, 15 Oct 2026 21:40:58 +0000 Subject: [PATCH 1/6] feat(datasource/zarr): read the offset chunk key encoding The `offset` encoding (zarr-extensions chunk-key-encodings/offset) stores a chunk under the `default` key of its grid coordinate plus a fixed per-dimension integer offset, so a writer can grow an array in the negative direction by whole chunks without renaming chunks. Parse the name and its `offset` (one integer per dimension, `/` separator by default), and add the offset where grid coordinates become keys. The key string composition moves out of the chunk source into `encodeChunkKey` so it can be unit-tested; `default` and `v2` keys are unchanged. Adds an example store that grew by one chunk row in front (keys c/-1/*) to the metadata snapshot tests. --- src/datasource/zarr/README.md | 8 + src/datasource/zarr/backend.ts | 18 +-- src/datasource/zarr/metadata/index.ts | 39 +++++ src/datasource/zarr/metadata/offset.spec.ts | 139 ++++++++++++++++++ src/datasource/zarr/metadata/parse.ts | 53 +++++-- .../zarr/zarr_v3/examples/offset_grown/c/-1/0 | Bin 0 -> 40 bytes .../zarr/zarr_v3/examples/offset_grown/c/-1/1 | Bin 0 -> 40 bytes .../zarr/zarr_v3/examples/offset_grown/c/0/0 | Bin 0 -> 40 bytes .../zarr/zarr_v3/examples/offset_grown/c/0/1 | Bin 0 -> 40 bytes .../zarr/zarr_v3/examples/offset_grown/c/1/0 | Bin 0 -> 40 bytes .../zarr/zarr_v3/examples/offset_grown/c/1/1 | Bin 0 -> 40 bytes .../zarr_v3/examples/offset_grown/zarr.json | 39 +++++ .../zarr_v3_examples_offset_grown.snapshot | 120 +++++++++++++++ tests/datasource/zarr.browser_test.ts | 1 + 14 files changed, 387 insertions(+), 30 deletions(-) create mode 100644 src/datasource/zarr/metadata/offset.spec.ts create mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/0 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/0 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/0 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/zarr.json create mode 100644 tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_offset_grown.snapshot diff --git a/src/datasource/zarr/README.md b/src/datasource/zarr/README.md index 1ff114f760..e99486afe5 100644 --- a/src/datasource/zarr/README.md +++ b/src/datasource/zarr/README.md @@ -54,6 +54,14 @@ Dimension names may be specified using an `_ARRAY_DIMENSIONS` attribute, ## Zarr v3 +Supported chunk key encodings: + +- default +- v2 +- offset (the `default` keys of the chunk grid coordinate plus a per-dimension offset, so an + array can grow in the negative direction without renaming chunks; keys may have negative + components) + Supported codecs: - crc32c (not validated) diff --git a/src/datasource/zarr/backend.ts b/src/datasource/zarr/backend.ts index 7370f7af0e..0bc66e5167 100644 --- a/src/datasource/zarr/backend.ts +++ b/src/datasource/zarr/backend.ts @@ -28,7 +28,7 @@ import { import "#src/datasource/zarr/codec/gzip/decode.js"; import "#src/datasource/zarr/codec/sharding_indexed/decode.js"; import "#src/datasource/zarr/codec/transpose/decode.js"; -import { ChunkKeyEncoding } from "#src/datasource/zarr/metadata/index.js"; +import { encodeChunkKey } from "#src/datasource/zarr/metadata/index.js"; import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; import { postProcessRawData } from "#src/sliceview/backend_chunk_decoders/postprocess.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; @@ -51,19 +51,8 @@ export class ZarrVolumeChunkSource extends WithParameters( const { parameters } = this; const { chunkGridPosition } = chunk; const { metadata } = parameters; - let baseKey = ""; const rank = this.spec.rank; const { physicalToLogicalDimension } = metadata.codecs.layoutInfo[0]; - let sep: string; - if (metadata.chunkKeyEncoding === ChunkKeyEncoding.DEFAULT) { - baseKey += "c"; - sep = metadata.dimensionSeparator; - } else { - sep = ""; - if (rank === 0) { - baseKey += "0"; - } - } const keyCoords = new Array(rank); const { readChunkShape } = metadata.codecs.layoutInfo[0]; const { chunkShape } = metadata; @@ -79,10 +68,7 @@ export class ZarrVolumeChunkSource extends WithParameters( chunkShape[decodedDim], ); } - for (let i = 0; i < rank; ++i) { - baseKey += `${sep}${keyCoords[i]}`; - sep = metadata.dimensionSeparator; - } + const baseKey = encodeChunkKey(metadata, keyCoords); const { chunkKvStore } = this; const response = await chunkKvStore.kvStore.read( chunkKvStore.getChunkKey(chunkGridPosition, baseKey), diff --git a/src/datasource/zarr/metadata/index.ts b/src/datasource/zarr/metadata/index.ts index 0cfae9bb96..19ad7d50b0 100644 --- a/src/datasource/zarr/metadata/index.ts +++ b/src/datasource/zarr/metadata/index.ts @@ -20,6 +20,9 @@ import type { DataType } from "#src/util/data_type.js"; export enum ChunkKeyEncoding { DEFAULT = 0, V2 = 1, + // `default` keys of `grid coordinate + chunkKeyOffset`: + // https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-key-encodings/offset + OFFSET = 2, } export type DimensionSeparator = "/" | "."; @@ -39,9 +42,45 @@ export interface ArrayMetadata { userAttributes: Record; dimensionSeparator: DimensionSeparator; chunkKeyEncoding: ChunkKeyEncoding; + // One entry per dimension under `ChunkKeyEncoding.OFFSET`, otherwise + // `undefined`. + chunkKeyOffset: number[] | undefined; codecs: CodecChainSpec; } +/** + * Formats the store key of the write chunk at `keyCoords` (one grid + * coordinate per logical dimension) according to `metadata`'s chunk key + * encoding. + */ +export function encodeChunkKey( + metadata: Pick< + ArrayMetadata, + "rank" | "chunkKeyEncoding" | "dimensionSeparator" | "chunkKeyOffset" + >, + keyCoords: ArrayLike, +): string { + const { rank, chunkKeyEncoding, dimensionSeparator, chunkKeyOffset } = + metadata; + let key = ""; + let sep: string; + if (chunkKeyEncoding === ChunkKeyEncoding.V2) { + sep = ""; + if (rank === 0) { + key += "0"; + } + } else { + key += "c"; + sep = dimensionSeparator; + } + for (let i = 0; i < rank; ++i) { + const offset = chunkKeyOffset === undefined ? 0 : chunkKeyOffset[i]; + key += `${sep}${keyCoords[i] + offset}`; + sep = dimensionSeparator; + } + return key; +} + export interface GroupMetadata { zarrVersion: 2 | 3; nodeType: "group"; diff --git a/src/datasource/zarr/metadata/offset.spec.ts b/src/datasource/zarr/metadata/offset.spec.ts new file mode 100644 index 0000000000..4d535aa327 --- /dev/null +++ b/src/datasource/zarr/metadata/offset.spec.ts @@ -0,0 +1,139 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, it } from "vitest"; +import type { ArrayMetadata } from "#src/datasource/zarr/metadata/index.js"; +import { + ChunkKeyEncoding, + encodeChunkKey, +} from "#src/datasource/zarr/metadata/index.js"; +import { parseV3Metadata } from "#src/datasource/zarr/metadata/parse.js"; +import "#src/datasource/zarr/codec/bytes/resolve.js"; + +function v3Array(chunkKeyEncoding: unknown) { + return { + zarr_format: 3, + node_type: "array", + shape: [6, 7], + data_type: "uint16", + chunk_grid: { name: "regular", configuration: { chunk_shape: [4, 5] } }, + chunk_key_encoding: chunkKeyEncoding, + fill_value: 0, + codecs: [{ name: "bytes", configuration: { endian: "little" } }], + }; +} + +function parseArray(chunkKeyEncoding: unknown): ArrayMetadata { + const metadata = parseV3Metadata(v3Array(chunkKeyEncoding), "array"); + if (metadata.nodeType !== "array") throw new Error("expected an array"); + return metadata; +} + +describe("offset chunk key encoding", () => { + it("parses the offset and defaults the separator to /", () => { + const metadata = parseArray({ + name: "offset", + configuration: { offset: [-2, 0] }, + }); + expect(metadata.chunkKeyEncoding).toBe(ChunkKeyEncoding.OFFSET); + expect(metadata.dimensionSeparator).toBe("/"); + expect(metadata.chunkKeyOffset).toEqual([-2, 0]); + }); + + it("accepts an explicit separator", () => { + const metadata = parseArray({ + name: "offset", + configuration: { separator: ".", offset: [1, -1] }, + }); + expect(metadata.dimensionSeparator).toBe("."); + expect(metadata.chunkKeyOffset).toEqual([1, -1]); + }); + + it("requires one integer offset per dimension", () => { + expect(() => parseArray({ name: "offset" })).toThrow(/offset/); + expect(() => + parseArray({ name: "offset", configuration: { offset: [1] } }), + ).toThrow(/length 2/); + expect(() => + parseArray({ name: "offset", configuration: { offset: [1, 0.5] } }), + ).toThrow(/integer/); + }); + + it("leaves the default and v2 encodings without an offset", () => { + expect(parseArray({ name: "default" }).chunkKeyOffset).toBeUndefined(); + expect(parseArray({ name: "v2" }).chunkKeyOffset).toBeUndefined(); + expect(() => parseArray({ name: "shifted" })).toThrow(/Invalid enum/); + }); +}); + +describe("encodeChunkKey", () => { + const base = { + rank: 2, + chunkKeyEncoding: ChunkKeyEncoding.DEFAULT, + dimensionSeparator: "/" as const, + chunkKeyOffset: undefined, + }; + + it("formats default and v2 keys as before", () => { + expect(encodeChunkKey(base, [3, 1])).toBe("c/3/1"); + expect(encodeChunkKey({ ...base, dimensionSeparator: "." }, [3, 1])).toBe( + "c.3.1", + ); + expect( + encodeChunkKey( + { + ...base, + chunkKeyEncoding: ChunkKeyEncoding.V2, + dimensionSeparator: ".", + }, + [3, 1], + ), + ).toBe("3.1"); + expect( + encodeChunkKey( + { ...base, rank: 0, chunkKeyEncoding: ChunkKeyEncoding.V2 }, + [], + ), + ).toBe("0"); + expect(encodeChunkKey({ ...base, rank: 0 }, [])).toBe("c"); + }); + + it("adds the offset per dimension, giving negative components in front", () => { + const offset = { + ...base, + chunkKeyEncoding: ChunkKeyEncoding.OFFSET, + chunkKeyOffset: [-2, 1], + }; + expect(encodeChunkKey(offset, [0, 0])).toBe("c/-2/1"); + expect(encodeChunkKey(offset, [2, 0])).toBe("c/0/1"); + expect(encodeChunkKey(offset, [5, 4])).toBe("c/3/5"); + }); + + it("with an all-zero offset matches the default encoding", () => { + const zero = { + ...base, + chunkKeyEncoding: ChunkKeyEncoding.OFFSET, + chunkKeyOffset: [0, 0], + }; + for (const coords of [ + [0, 0], + [3, 1], + [12, 7], + ]) { + expect(encodeChunkKey(zero, coords)).toBe(encodeChunkKey(base, coords)); + } + }); +}); diff --git a/src/datasource/zarr/metadata/parse.ts b/src/datasource/zarr/metadata/parse.ts index c77e710772..2536a2de9f 100644 --- a/src/datasource/zarr/metadata/parse.ts +++ b/src/datasource/zarr/metadata/parse.ts @@ -62,6 +62,15 @@ export function parseChunkShape(obj: unknown, rank: number): number[] { }); } +function verifyStrictInt(value: unknown): number { + if (!Number.isInteger(value)) { + throw new Error( + `Expected integer, but received: ${JSON.stringify(value)}.`, + ); + } + return value as number; +} + export function parseDimensionSeparator(value: unknown): "/" | "." { if (value !== "." && value !== "/") { throw new Error( @@ -256,20 +265,34 @@ export function parseV3Metadata( }, ); - const { configuration: dimensionSeparator, name: chunkKeyEncoding } = - verifyObjectProperty(obj, "chunk_key_encoding", (value) => - parseNameAndConfiguration( - value, - (name) => verifyEnumString(name, ChunkKeyEncoding, /^(v2|default)$/), - (configuration, chunkKeyEncoding) => - verifyOptionalObjectProperty( - configuration, - "separator", - parseDimensionSeparator, - chunkKeyEncoding === ChunkKeyEncoding.DEFAULT ? "/" : ".", - ), - ), - ); + const { + configuration: { dimensionSeparator, chunkKeyOffset }, + name: chunkKeyEncoding, + } = verifyObjectProperty(obj, "chunk_key_encoding", (value) => + parseNameAndConfiguration( + value, + (name) => + verifyEnumString(name, ChunkKeyEncoding, /^(v2|default|offset)$/), + (configuration, chunkKeyEncoding) => ({ + dimensionSeparator: verifyOptionalObjectProperty( + configuration, + "separator", + parseDimensionSeparator, + chunkKeyEncoding === ChunkKeyEncoding.V2 ? "." : "/", + ), + chunkKeyOffset: + chunkKeyEncoding === ChunkKeyEncoding.OFFSET + ? verifyObjectProperty(configuration, "offset", (offset) => + parseFixedLengthArray( + new Array(rank), + offset, + verifyStrictInt, + ), + ) + : undefined, + }), + ), + ); const fillValue = verifyObjectProperty(obj, "fill_value", (value) => parseFillValue(dataType, value), @@ -291,6 +314,7 @@ export function parseV3Metadata( dimensionUnits, chunkKeyEncoding, dimensionSeparator, + chunkKeyOffset, userAttributes, codecs, }; @@ -436,6 +460,7 @@ export function parseV2Metadata( userAttributes: attrs, dimensionSeparator, chunkKeyEncoding: ChunkKeyEncoding.V2, + chunkKeyOffset: undefined, codecs: codecChainSpec, }; } catch (e) { diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/0 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/0 new file mode 100644 index 0000000000000000000000000000000000000000..8a807b6f8b3b33753ff0f12843a31f81ecd760c0 GIT binary patch literal 40 qcmV~$feC;h006LA2AP7UVA=nNwgsDAbT}oKtf-n>H%yP$^7#P|C;<-u literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/1 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/1 new file mode 100644 index 0000000000000000000000000000000000000000..c8768a8563e68f2e47ee18b23e0de2a9bd8f168b GIT binary patch literal 40 acmZQ$U}t~;UIsoWON2ob%93V~fwBM!C;*87 literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/0 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..7abc66d5c8bf7c58ccd4328aef8255a8dc8e6bda GIT binary patch literal 40 qcmV~$feC;h006KZYZ+t;nu2A&w*{MBbT}oKtf-n>H%yP$^7#R=lnBcJ literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/1 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/1 new file mode 100644 index 0000000000000000000000000000000000000000..83836c01fb8a12a72ff85cc0207256829301b4dc GIT binary patch literal 40 acmc~~$YOwi0)|2;tDK<%%Bo|ihq3@vXak`D literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/0 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/0 new file mode 100644 index 0000000000000000000000000000000000000000..d969e65c85b8c01d612a2d717db95d8e54dd14e7 GIT binary patch literal 40 ccmZo*Xk=(&Xl7_(XlLkP=w#?(=w`qM0HFE>H~;_u literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/1 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/1 new file mode 100644 index 0000000000000000000000000000000000000000..015e4c72ffc1e158213774b60564fae09ba5338c GIT binary patch literal 40 TcmZo=Xk&nY9)?~h3mXLhNT>oO literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/zarr.json b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/zarr.json new file mode 100644 index 0000000000..315eda61ea --- /dev/null +++ b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/zarr.json @@ -0,0 +1,39 @@ +{ + "shape": [ + 10, + 7 + ], + "data_type": "uint16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 4, + 5 + ] + } + }, + "chunk_key_encoding": { + "name": "offset", + "configuration": { + "separator": "/", + "offset": [ + -1, + 0 + ] + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "attributes": {}, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_offset_grown.snapshot b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_offset_grown.snapshot new file mode 100644 index 0000000000..5438209024 --- /dev/null +++ b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_offset_grown.snapshot @@ -0,0 +1,120 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/zarr/zarr_v3/examples/offset_grown/|zarr3:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + 0, + 0, + ], + "upperBounds": Float64Array [ + 10, + 7, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "dim_0", + "dim_1", + ], + "scales": Float64Array [ + 1, + 1, + ], + "units": [ + "", + "", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/zarr_v3/examples/offset_grown/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 4, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 0, + 0, + ], + "upperVoxelBound": [ + 7, + 10, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 1, + 0, + ], + [ + 1, + 0, + 0, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + 0, + 0, + ], + "pointB": [ + 10, + 7, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/zarr.browser_test.ts b/tests/datasource/zarr.browser_test.ts index b3ebcb33fd..ec76b3733e 100644 --- a/tests/datasource/zarr.browser_test.ts +++ b/tests/datasource/zarr.browser_test.ts @@ -24,6 +24,7 @@ import { datasourceMetadataSnapshotTests } from "#tests/datasource/metadata_snap datasourceMetadataSnapshotTests("zarr", [ "zarr_v3/examples/single_res", "zarr_v3/examples/transpose", + "zarr_v3/examples/offset_grown", "ome_zarr/simple_0.4", "ome_zarr/simple_0.5", "ome_zarr/simple_0.5.zip", From 9f65b75e9c30bea42e311d70ed6bdd2ac6307b5b Mon Sep 17 00:00:00 2001 From: Jeff Rhoades Date: Thu, 15 Oct 2026 22:31:15 +0000 Subject: [PATCH 2/6] zarr: shorten the offset encoding comments --- src/datasource/zarr/metadata/index.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/datasource/zarr/metadata/index.ts b/src/datasource/zarr/metadata/index.ts index 19ad7d50b0..ed2ecc8ee0 100644 --- a/src/datasource/zarr/metadata/index.ts +++ b/src/datasource/zarr/metadata/index.ts @@ -20,7 +20,6 @@ import type { DataType } from "#src/util/data_type.js"; export enum ChunkKeyEncoding { DEFAULT = 0, V2 = 1, - // `default` keys of `grid coordinate + chunkKeyOffset`: // https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-key-encodings/offset OFFSET = 2, } @@ -42,17 +41,12 @@ export interface ArrayMetadata { userAttributes: Record; dimensionSeparator: DimensionSeparator; chunkKeyEncoding: ChunkKeyEncoding; - // One entry per dimension under `ChunkKeyEncoding.OFFSET`, otherwise - // `undefined`. + // One entry per dimension under `ChunkKeyEncoding.OFFSET`. chunkKeyOffset: number[] | undefined; codecs: CodecChainSpec; } -/** - * Formats the store key of the write chunk at `keyCoords` (one grid - * coordinate per logical dimension) according to `metadata`'s chunk key - * encoding. - */ +/** Store key of the write chunk at grid coordinates `keyCoords`. */ export function encodeChunkKey( metadata: Pick< ArrayMetadata, From 1e8044cc12ed563949c69e0409939c228c3cf81d Mon Sep 17 00:00:00 2001 From: Jeff Rhoades Date: Thu, 15 Oct 2026 23:12:32 +0000 Subject: [PATCH 3/6] zarr: use optional chaining for the chunk key offset --- src/datasource/zarr/metadata/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasource/zarr/metadata/index.ts b/src/datasource/zarr/metadata/index.ts index ed2ecc8ee0..6b20115742 100644 --- a/src/datasource/zarr/metadata/index.ts +++ b/src/datasource/zarr/metadata/index.ts @@ -68,7 +68,7 @@ export function encodeChunkKey( sep = dimensionSeparator; } for (let i = 0; i < rank; ++i) { - const offset = chunkKeyOffset === undefined ? 0 : chunkKeyOffset[i]; + const offset = chunkKeyOffset?.[i] ?? 0; key += `${sep}${keyCoords[i] + offset}`; sep = dimensionSeparator; } From b9e8ead50ebba7307749e43e6287e789f3fc040c Mon Sep 17 00:00:00 2001 From: Jeff Rhoades Date: Fri, 16 Oct 2026 23:03:33 +0000 Subject: [PATCH 4/6] feat(datasource/zarr): read the regular_bounded chunk grid Rework following zarr-extensions#79: the `offset` chunk key encoding is removed. `regular_bounded` is a regular chunk grid with an `origin` (the array coordinate where chunk 0 starts) and a `lower_bound` (the coordinate of the first element), so chunk grid indices may be negative and the first chunk may be partly filled. Chunk coordinates are taken relative to the origin, as the precomputed source does with its voxel offset: the voxel bounds are `[lower_bound - origin, lower_bound - origin + shape)`, the origin is added in the chunk-to-multiscale transform, and the model bounds start at `lower_bound`. Grid positions, and so keys, come out negative where they should without any change to key formatting. Two example stores written by zarr-python are added to the metadata snapshot tests: one grown in front by `Array.prepend`, and one with a non-zero origin on both axes. tensorstore reads both identically. --- src/datasource/zarr/README.md | 10 +- src/datasource/zarr/backend.ts | 18 ++- src/datasource/zarr/frontend.ts | 25 +++- src/datasource/zarr/metadata/index.ts | 37 +---- src/datasource/zarr/metadata/offset.spec.ts | 139 ------------------ src/datasource/zarr/metadata/parse.ts | 99 +++++++------ .../zarr/metadata/regular_bounded.spec.ts | 86 +++++++++++ .../zarr/zarr_v3/examples/offset_grown/c/-1/0 | Bin 40 -> 0 bytes .../zarr/zarr_v3/examples/offset_grown/c/-1/1 | Bin 40 -> 0 bytes .../zarr_v3/examples/regular_bounded/c/-1/-1 | Bin 0 -> 40 bytes .../zarr_v3/examples/regular_bounded/c/-1/0 | Bin 0 -> 40 bytes .../zarr_v3/examples/regular_bounded/c/-1/1 | Bin 0 -> 40 bytes .../zarr_v3/examples/regular_bounded/c/0/-1 | Bin 0 -> 40 bytes .../{offset_grown => regular_bounded}/c/0/0 | Bin .../{offset_grown => regular_bounded}/c/0/1 | Bin .../zarr_v3/examples/regular_bounded/c/1/-1 | Bin 0 -> 40 bytes .../{offset_grown => regular_bounded}/c/1/0 | Bin .../{offset_grown => regular_bounded}/c/1/1 | Bin .../zarr.json | 22 +-- .../examples/regular_bounded_origin/c/-1/0 | Bin 0 -> 12 bytes .../examples/regular_bounded_origin/c/-1/1 | Bin 0 -> 12 bytes .../examples/regular_bounded_origin/c/0/0 | Bin 0 -> 12 bytes .../examples/regular_bounded_origin/c/0/1 | Bin 0 -> 12 bytes .../examples/regular_bounded_origin/zarr.json | 43 ++++++ .../zarr/zarr_v3/generate_regular_bounded.py | 35 +++++ ...zarr_v3_examples_regular_bounded.snapshot} | 22 +-- ...3_examples_regular_bounded_origin.snapshot | 120 +++++++++++++++ tests/datasource/zarr.browser_test.ts | 3 +- 28 files changed, 409 insertions(+), 250 deletions(-) delete mode 100644 src/datasource/zarr/metadata/offset.spec.ts create mode 100644 src/datasource/zarr/metadata/regular_bounded.spec.ts delete mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/0 delete mode 100644 testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/-1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/0 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/0/-1 rename testdata/datasource/zarr/zarr_v3/examples/{offset_grown => regular_bounded}/c/0/0 (100%) rename testdata/datasource/zarr/zarr_v3/examples/{offset_grown => regular_bounded}/c/0/1 (100%) create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/1/-1 rename testdata/datasource/zarr/zarr_v3/examples/{offset_grown => regular_bounded}/c/1/0 (100%) rename testdata/datasource/zarr/zarr_v3/examples/{offset_grown => regular_bounded}/c/1/1 (100%) rename testdata/datasource/zarr/zarr_v3/examples/{offset_grown => regular_bounded}/zarr.json (70%) create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/-1/0 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/-1/1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/0/0 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/0/1 create mode 100644 testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/zarr.json create mode 100644 testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py rename tests/datasource/metadata_snapshots/zarr/{zarr_v3_examples_offset_grown.snapshot => zarr_v3_examples_regular_bounded.snapshot} (90%) create mode 100644 tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_regular_bounded_origin.snapshot diff --git a/src/datasource/zarr/README.md b/src/datasource/zarr/README.md index e99486afe5..c3874833fd 100644 --- a/src/datasource/zarr/README.md +++ b/src/datasource/zarr/README.md @@ -54,13 +54,11 @@ Dimension names may be specified using an `_ARRAY_DIMENSIONS` attribute, ## Zarr v3 -Supported chunk key encodings: +Supported chunk grids: -- default -- v2 -- offset (the `default` keys of the chunk grid coordinate plus a per-dimension offset, so an - array can grow in the negative direction without renaming chunks; keys may have negative - components) +- regular +- regular_bounded (a regular grid with an `origin` and a `lower_bound`, under which chunk keys + may be negative and the first chunk may be partly filled) Supported codecs: diff --git a/src/datasource/zarr/backend.ts b/src/datasource/zarr/backend.ts index 0bc66e5167..7370f7af0e 100644 --- a/src/datasource/zarr/backend.ts +++ b/src/datasource/zarr/backend.ts @@ -28,7 +28,7 @@ import { import "#src/datasource/zarr/codec/gzip/decode.js"; import "#src/datasource/zarr/codec/sharding_indexed/decode.js"; import "#src/datasource/zarr/codec/transpose/decode.js"; -import { encodeChunkKey } from "#src/datasource/zarr/metadata/index.js"; +import { ChunkKeyEncoding } from "#src/datasource/zarr/metadata/index.js"; import { WithSharedKvStoreContextCounterpart } from "#src/kvstore/backend.js"; import { postProcessRawData } from "#src/sliceview/backend_chunk_decoders/postprocess.js"; import type { VolumeChunk } from "#src/sliceview/volume/backend.js"; @@ -51,8 +51,19 @@ export class ZarrVolumeChunkSource extends WithParameters( const { parameters } = this; const { chunkGridPosition } = chunk; const { metadata } = parameters; + let baseKey = ""; const rank = this.spec.rank; const { physicalToLogicalDimension } = metadata.codecs.layoutInfo[0]; + let sep: string; + if (metadata.chunkKeyEncoding === ChunkKeyEncoding.DEFAULT) { + baseKey += "c"; + sep = metadata.dimensionSeparator; + } else { + sep = ""; + if (rank === 0) { + baseKey += "0"; + } + } const keyCoords = new Array(rank); const { readChunkShape } = metadata.codecs.layoutInfo[0]; const { chunkShape } = metadata; @@ -68,7 +79,10 @@ export class ZarrVolumeChunkSource extends WithParameters( chunkShape[decodedDim], ); } - const baseKey = encodeChunkKey(metadata, keyCoords); + for (let i = 0; i < rank; ++i) { + baseKey += `${sep}${keyCoords[i]}`; + sep = metadata.dimensionSeparator; + } const { chunkKvStore } = this; const response = await chunkKvStore.kvStore.read( chunkKvStore.getChunkKey(chunkGridPosition, baseKey), diff --git a/src/datasource/zarr/frontend.ts b/src/datasource/zarr/frontend.ts index 9197cdebe5..fd9342c51b 100644 --- a/src/datasource/zarr/frontend.ts +++ b/src/datasource/zarr/frontend.ts @@ -119,18 +119,23 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou return transposeNestedArrays( this.multiscale.scales.map((scale) => { const { metadata } = scale; - const { rank, codecs, shape } = metadata; + const { rank, codecs, shape, chunkGridOrigin, lowerBound } = metadata; const readChunkShape = codecs.layoutInfo[0].readChunkShape; const { physicalToLogicalDimension } = metadata.codecs.layoutInfo[0]; const permutedChunkShape = new Uint32Array(rank); - const permutedDataShape = new Float32Array(rank); + const permutedLowerBound = new Float32Array(rank); + const permutedUpperBound = new Float32Array(rank); const orderTransform = new Float32Array((rank + 1) ** 2); orderTransform[(rank + 1) ** 2 - 1] = 1; + // Chunk coordinates are relative to the chunk grid origin. for (let i = 0; i < rank; ++i) { const decodedDim = physicalToLogicalDimension[rank - 1 - i]; permutedChunkShape[i] = readChunkShape[decodedDim]; - permutedDataShape[i] = shape[decodedDim]; + permutedLowerBound[i] = + lowerBound[decodedDim] - chunkGridOrigin[decodedDim]; + permutedUpperBound[i] = permutedLowerBound[i] + shape[decodedDim]; orderTransform[decodedDim + i * (rank + 1)] = 1; + orderTransform[(rank + 1) * rank + i] = chunkGridOrigin[i]; } const transform = new Float32Array((rank + 1) ** 2); matrix.multiply( @@ -148,7 +153,8 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou rank, chunkToMultiscaleTransform: transform, dataType: metadata.dataType, - upperVoxelBound: permutedDataShape, + lowerVoxelBound: permutedLowerBound, + upperVoxelBound: permutedUpperBound, volumeType: this.volumeType, chunkDataSizes: [permutedChunkShape], volumeSourceOptions, @@ -275,8 +281,11 @@ function getMultiscaleInfoForSingleArray( units: Array.from(unitsAndScales, (x) => x.unit), boundingBoxes: [ makeIdentityTransformedBoundingBox({ - lowerBounds: new Float64Array(metadata.rank), - upperBounds: Float64Array.from(metadata.shape), + lowerBounds: Float64Array.from(metadata.lowerBound), + upperBounds: Float64Array.from( + metadata.shape, + (n, i) => metadata.lowerBound[i] + n, + ), }), ], }); @@ -347,7 +356,9 @@ async function resolveOmeMultiscale( const baseScale = multiscale.scales[0]; const baseZarrMetadata = scaleZarrMetadata[0]; for (let i = 0; i < rank; ++i) { - const lower = (lowerBounds[i] = baseScale.transform[(rank + 1) * rank + i]); + const lower = (lowerBounds[i] = + baseScale.transform[(rank + 1) * rank + i] + + baseZarrMetadata.lowerBound[i]); upperBounds[i] = lower + baseZarrMetadata.shape[i]; } const boundingBox = makeIdentityTransformedBoundingBox({ diff --git a/src/datasource/zarr/metadata/index.ts b/src/datasource/zarr/metadata/index.ts index 6b20115742..b22912e582 100644 --- a/src/datasource/zarr/metadata/index.ts +++ b/src/datasource/zarr/metadata/index.ts @@ -20,8 +20,6 @@ import type { DataType } from "#src/util/data_type.js"; export enum ChunkKeyEncoding { DEFAULT = 0, V2 = 1, - // https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-key-encodings/offset - OFFSET = 2, } export type DimensionSeparator = "/" | "."; @@ -41,40 +39,13 @@ export interface ArrayMetadata { userAttributes: Record; dimensionSeparator: DimensionSeparator; chunkKeyEncoding: ChunkKeyEncoding; - // One entry per dimension under `ChunkKeyEncoding.OFFSET`. - chunkKeyOffset: number[] | undefined; + // Array coordinate where chunk 0 starts, and coordinate of the first + // element; all zero except under the `regular_bounded` chunk grid. + chunkGridOrigin: number[]; + lowerBound: number[]; codecs: CodecChainSpec; } -/** Store key of the write chunk at grid coordinates `keyCoords`. */ -export function encodeChunkKey( - metadata: Pick< - ArrayMetadata, - "rank" | "chunkKeyEncoding" | "dimensionSeparator" | "chunkKeyOffset" - >, - keyCoords: ArrayLike, -): string { - const { rank, chunkKeyEncoding, dimensionSeparator, chunkKeyOffset } = - metadata; - let key = ""; - let sep: string; - if (chunkKeyEncoding === ChunkKeyEncoding.V2) { - sep = ""; - if (rank === 0) { - key += "0"; - } - } else { - key += "c"; - sep = dimensionSeparator; - } - for (let i = 0; i < rank; ++i) { - const offset = chunkKeyOffset?.[i] ?? 0; - key += `${sep}${keyCoords[i] + offset}`; - sep = dimensionSeparator; - } - return key; -} - export interface GroupMetadata { zarrVersion: 2 | 3; nodeType: "group"; diff --git a/src/datasource/zarr/metadata/offset.spec.ts b/src/datasource/zarr/metadata/offset.spec.ts deleted file mode 100644 index 4d535aa327..0000000000 --- a/src/datasource/zarr/metadata/offset.spec.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * @license - * Copyright 2026 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { describe, expect, it } from "vitest"; -import type { ArrayMetadata } from "#src/datasource/zarr/metadata/index.js"; -import { - ChunkKeyEncoding, - encodeChunkKey, -} from "#src/datasource/zarr/metadata/index.js"; -import { parseV3Metadata } from "#src/datasource/zarr/metadata/parse.js"; -import "#src/datasource/zarr/codec/bytes/resolve.js"; - -function v3Array(chunkKeyEncoding: unknown) { - return { - zarr_format: 3, - node_type: "array", - shape: [6, 7], - data_type: "uint16", - chunk_grid: { name: "regular", configuration: { chunk_shape: [4, 5] } }, - chunk_key_encoding: chunkKeyEncoding, - fill_value: 0, - codecs: [{ name: "bytes", configuration: { endian: "little" } }], - }; -} - -function parseArray(chunkKeyEncoding: unknown): ArrayMetadata { - const metadata = parseV3Metadata(v3Array(chunkKeyEncoding), "array"); - if (metadata.nodeType !== "array") throw new Error("expected an array"); - return metadata; -} - -describe("offset chunk key encoding", () => { - it("parses the offset and defaults the separator to /", () => { - const metadata = parseArray({ - name: "offset", - configuration: { offset: [-2, 0] }, - }); - expect(metadata.chunkKeyEncoding).toBe(ChunkKeyEncoding.OFFSET); - expect(metadata.dimensionSeparator).toBe("/"); - expect(metadata.chunkKeyOffset).toEqual([-2, 0]); - }); - - it("accepts an explicit separator", () => { - const metadata = parseArray({ - name: "offset", - configuration: { separator: ".", offset: [1, -1] }, - }); - expect(metadata.dimensionSeparator).toBe("."); - expect(metadata.chunkKeyOffset).toEqual([1, -1]); - }); - - it("requires one integer offset per dimension", () => { - expect(() => parseArray({ name: "offset" })).toThrow(/offset/); - expect(() => - parseArray({ name: "offset", configuration: { offset: [1] } }), - ).toThrow(/length 2/); - expect(() => - parseArray({ name: "offset", configuration: { offset: [1, 0.5] } }), - ).toThrow(/integer/); - }); - - it("leaves the default and v2 encodings without an offset", () => { - expect(parseArray({ name: "default" }).chunkKeyOffset).toBeUndefined(); - expect(parseArray({ name: "v2" }).chunkKeyOffset).toBeUndefined(); - expect(() => parseArray({ name: "shifted" })).toThrow(/Invalid enum/); - }); -}); - -describe("encodeChunkKey", () => { - const base = { - rank: 2, - chunkKeyEncoding: ChunkKeyEncoding.DEFAULT, - dimensionSeparator: "/" as const, - chunkKeyOffset: undefined, - }; - - it("formats default and v2 keys as before", () => { - expect(encodeChunkKey(base, [3, 1])).toBe("c/3/1"); - expect(encodeChunkKey({ ...base, dimensionSeparator: "." }, [3, 1])).toBe( - "c.3.1", - ); - expect( - encodeChunkKey( - { - ...base, - chunkKeyEncoding: ChunkKeyEncoding.V2, - dimensionSeparator: ".", - }, - [3, 1], - ), - ).toBe("3.1"); - expect( - encodeChunkKey( - { ...base, rank: 0, chunkKeyEncoding: ChunkKeyEncoding.V2 }, - [], - ), - ).toBe("0"); - expect(encodeChunkKey({ ...base, rank: 0 }, [])).toBe("c"); - }); - - it("adds the offset per dimension, giving negative components in front", () => { - const offset = { - ...base, - chunkKeyEncoding: ChunkKeyEncoding.OFFSET, - chunkKeyOffset: [-2, 1], - }; - expect(encodeChunkKey(offset, [0, 0])).toBe("c/-2/1"); - expect(encodeChunkKey(offset, [2, 0])).toBe("c/0/1"); - expect(encodeChunkKey(offset, [5, 4])).toBe("c/3/5"); - }); - - it("with an all-zero offset matches the default encoding", () => { - const zero = { - ...base, - chunkKeyEncoding: ChunkKeyEncoding.OFFSET, - chunkKeyOffset: [0, 0], - }; - for (const coords of [ - [0, 0], - [3, 1], - [12, 7], - ]) { - expect(encodeChunkKey(zero, coords)).toBe(encodeChunkKey(base, coords)); - } - }); -}); diff --git a/src/datasource/zarr/metadata/parse.ts b/src/datasource/zarr/metadata/parse.ts index 2536a2de9f..3d69996c01 100644 --- a/src/datasource/zarr/metadata/parse.ts +++ b/src/datasource/zarr/metadata/parse.ts @@ -62,7 +62,12 @@ export function parseChunkShape(obj: unknown, rank: number): number[] { }); } -function verifyStrictInt(value: unknown): number { +enum ChunkGrid { + REGULAR = 0, + REGULAR_BOUNDED = 1, +} + +function verifyInteger(value: unknown): number { if (!Number.isInteger(value)) { throw new Error( `Expected integer, but received: ${JSON.stringify(value)}.`, @@ -234,18 +239,40 @@ export function parseV3Metadata( verifyEnumString(x, DataType, /^[a-z0-9]+$/), ); - const { configuration: chunkShape } = verifyObjectProperty( - obj, - "chunk_grid", - (chunkGrid) => - parseNameAndConfiguration( - chunkGrid, - (name) => verifyConstant(name, "regular"), - (configuration) => - verifyObjectProperty(configuration, "chunk_shape", (chunks) => - parseChunkShape(chunks, rank), + const { + configuration: { chunkShape, chunkGridOrigin, lowerBound }, + } = verifyObjectProperty(obj, "chunk_grid", (chunkGrid) => + parseNameAndConfiguration( + chunkGrid, + (name) => + verifyEnumString(name, ChunkGrid, /^(regular|regular_bounded)$/), + (configuration, name) => { + const chunkShape = verifyObjectProperty( + configuration, + "chunk_shape", + (chunks) => parseChunkShape(chunks, rank), + ); + if (name === ChunkGrid.REGULAR) { + const zeros = new Array(rank).fill(0); + return { chunkShape, chunkGridOrigin: zeros, lowerBound: zeros }; + } + const parseCoordinates = (x: unknown) => + parseFixedLengthArray(new Array(rank), x, verifyInteger); + return { + chunkShape, + chunkGridOrigin: verifyObjectProperty( + configuration, + "origin", + parseCoordinates, ), - ), + lowerBound: verifyObjectProperty( + configuration, + "lower_bound", + parseCoordinates, + ), + }; + }, + ), ); const { userAttributes, dimensionUnits } = verifyObjectProperty( @@ -265,34 +292,20 @@ export function parseV3Metadata( }, ); - const { - configuration: { dimensionSeparator, chunkKeyOffset }, - name: chunkKeyEncoding, - } = verifyObjectProperty(obj, "chunk_key_encoding", (value) => - parseNameAndConfiguration( - value, - (name) => - verifyEnumString(name, ChunkKeyEncoding, /^(v2|default|offset)$/), - (configuration, chunkKeyEncoding) => ({ - dimensionSeparator: verifyOptionalObjectProperty( - configuration, - "separator", - parseDimensionSeparator, - chunkKeyEncoding === ChunkKeyEncoding.V2 ? "." : "/", - ), - chunkKeyOffset: - chunkKeyEncoding === ChunkKeyEncoding.OFFSET - ? verifyObjectProperty(configuration, "offset", (offset) => - parseFixedLengthArray( - new Array(rank), - offset, - verifyStrictInt, - ), - ) - : undefined, - }), - ), - ); + const { configuration: dimensionSeparator, name: chunkKeyEncoding } = + verifyObjectProperty(obj, "chunk_key_encoding", (value) => + parseNameAndConfiguration( + value, + (name) => verifyEnumString(name, ChunkKeyEncoding, /^(v2|default)$/), + (configuration, chunkKeyEncoding) => + verifyOptionalObjectProperty( + configuration, + "separator", + parseDimensionSeparator, + chunkKeyEncoding === ChunkKeyEncoding.DEFAULT ? "/" : ".", + ), + ), + ); const fillValue = verifyObjectProperty(obj, "fill_value", (value) => parseFillValue(dataType, value), @@ -314,7 +327,8 @@ export function parseV3Metadata( dimensionUnits, chunkKeyEncoding, dimensionSeparator, - chunkKeyOffset, + chunkGridOrigin, + lowerBound, userAttributes, codecs, }; @@ -460,7 +474,8 @@ export function parseV2Metadata( userAttributes: attrs, dimensionSeparator, chunkKeyEncoding: ChunkKeyEncoding.V2, - chunkKeyOffset: undefined, + chunkGridOrigin: new Array(rank).fill(0), + lowerBound: new Array(rank).fill(0), codecs: codecChainSpec, }; } catch (e) { diff --git a/src/datasource/zarr/metadata/regular_bounded.spec.ts b/src/datasource/zarr/metadata/regular_bounded.spec.ts new file mode 100644 index 0000000000..07c50087e8 --- /dev/null +++ b/src/datasource/zarr/metadata/regular_bounded.spec.ts @@ -0,0 +1,86 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, expect, it } from "vitest"; +import "#src/datasource/zarr/codec/bytes/resolve.js"; +import type { ArrayMetadata } from "#src/datasource/zarr/metadata/index.js"; +import { parseV3Metadata } from "#src/datasource/zarr/metadata/parse.js"; + +function parseArray(chunkGrid: unknown): ArrayMetadata { + const metadata = parseV3Metadata( + { + zarr_format: 3, + node_type: "array", + shape: [6, 7], + data_type: "uint16", + chunk_grid: chunkGrid, + chunk_key_encoding: { name: "default" }, + fill_value: 0, + codecs: [{ name: "bytes", configuration: { endian: "little" } }], + }, + "array", + ); + if (metadata.nodeType !== "array") throw new Error("expected an array"); + return metadata; +} + +describe("regular_bounded chunk grid", () => { + it("parses the origin and lower bound", () => { + const metadata = parseArray({ + name: "regular_bounded", + configuration: { + chunk_shape: [4, 5], + origin: [1, 0], + lower_bound: [-3, -2], + }, + }); + expect(metadata.chunkShape).toEqual([4, 5]); + expect(metadata.chunkGridOrigin).toEqual([1, 0]); + expect(metadata.lowerBound).toEqual([-3, -2]); + }); + + it("leaves a regular grid at zero", () => { + const metadata = parseArray({ + name: "regular", + configuration: { chunk_shape: [4, 5] }, + }); + expect(metadata.chunkGridOrigin).toEqual([0, 0]); + expect(metadata.lowerBound).toEqual([0, 0]); + }); + + it("requires one integer per dimension for both fields", () => { + const grid = (configuration: object) => ({ + name: "regular_bounded", + configuration: { chunk_shape: [4, 5], ...configuration }, + }); + expect(() => parseArray(grid({ origin: [0, 0] }))).toThrow(/lower_bound/); + expect(() => + parseArray(grid({ origin: [0], lower_bound: [0, 0] })), + ).toThrow(/length 2/); + expect(() => + parseArray(grid({ origin: [0, 0], lower_bound: [0, 0.5] })), + ).toThrow(/integer/); + }); + + it("refuses an unknown chunk grid", () => { + expect(() => + parseArray({ + name: "regular_offset", + configuration: { chunk_shape: [4, 5] }, + }), + ).toThrow(/Invalid enum/); + }); +}); diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/0 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/0 deleted file mode 100644 index 8a807b6f8b3b33753ff0f12843a31f81ecd760c0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40 qcmV~$feC;h006LA2AP7UVA=nNwgsDAbT}oKtf-n>H%yP$^7#P|C;<-u diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/1 b/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/-1/1 deleted file mode 100644 index c8768a8563e68f2e47ee18b23e0de2a9bd8f168b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40 acmZQ$U}t~;UIsoWON2ob%93V~fwBM!C;*87 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/-1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/-1 new file mode 100644 index 0000000000000000000000000000000000000000..df77d40c30569bdb911702bbdb3b575e73aa4d38 GIT binary patch literal 40 ScmZQzKm{$#EzA%RR2Bd!I0GvH literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/0 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/0 new file mode 100644 index 0000000000000000000000000000000000000000..1b807ba68bc1d819559aa8ecd09aba400d01101c GIT binary patch literal 40 lcmXxRM+(3o006MKzy)gi|G#=N;R&K7E2^d&re!-mzY7MC0G9v& literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/-1/1 new file mode 100644 index 0000000000000000000000000000000000000000..5d26c145111c0d570b962a56abd53f1c9f0fd6a0 GIT binary patch literal 40 YcmZQzfB`lJb|{mVfe*?OVGxC~00j^LQUCw| literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/0/-1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/0/-1 new file mode 100644 index 0000000000000000000000000000000000000000..59f9272f158db34c5217cff4e5601582a4705e9c GIT binary patch literal 40 ScmZQz0D%_f7G^L5Ckp^f00cPz literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/0 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/0/0 similarity index 100% rename from testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/0 rename to testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/0/0 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/0/1 similarity index 100% rename from testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/0/1 rename to testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/0/1 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/1/-1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/1/-1 new file mode 100644 index 0000000000000000000000000000000000000000..1a663dbdc2a0c192f47119f01ad708a7c781a3ca GIT binary patch literal 40 VcmZQz0D%_f7G^L5i3Q=qNB}!50v!MV literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/0 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/1/0 similarity index 100% rename from testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/0 rename to testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/1/0 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/1/1 similarity index 100% rename from testdata/datasource/zarr/zarr_v3/examples/offset_grown/c/1/1 rename to testdata/datasource/zarr/zarr_v3/examples/regular_bounded/c/1/1 diff --git a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/zarr.json b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/zarr.json similarity index 70% rename from testdata/datasource/zarr/zarr_v3/examples/offset_grown/zarr.json rename to testdata/datasource/zarr/zarr_v3/examples/regular_bounded/zarr.json index 315eda61ea..d07af7c355 100644 --- a/testdata/datasource/zarr/zarr_v3/examples/offset_grown/zarr.json +++ b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded/zarr.json @@ -1,26 +1,30 @@ { "shape": [ - 10, - 7 + 9, + 9 ], "data_type": "uint16", "chunk_grid": { - "name": "regular", + "name": "regular_bounded", "configuration": { "chunk_shape": [ 4, 5 + ], + "origin": [ + 0, + 0 + ], + "lower_bound": [ + -3, + -2 ] } }, "chunk_key_encoding": { - "name": "offset", + "name": "default", "configuration": { - "separator": "/", - "offset": [ - -1, - 0 - ] + "separator": "/" } }, "fill_value": 0, diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/-1/0 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/-1/0 new file mode 100644 index 0000000000000000000000000000000000000000..0e79de19f5858fd2e225a85eef86fb2cfb223a4f GIT binary patch literal 12 TcmZQzU}RuoU|?WnU}FFP07d{0 literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/-1/1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/-1/1 new file mode 100644 index 0000000000000000000000000000000000000000..59d563b714fadb447a6b2da622266a102feb0e20 GIT binary patch literal 12 RcmZQ(U}0cjU}xX}k^lgr02crN literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/0/0 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..f0672235874b883dbaff20b51b466207de359248 GIT binary patch literal 12 OcmZQz;AG%pfB*mhwg40W literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/0/1 b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/c/0/1 new file mode 100644 index 0000000000000000000000000000000000000000..9fd4d412843e1d56d48352d89c172f31b17b4045 GIT binary patch literal 12 Mcmd;P;9-CP009U97ytkO literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/zarr.json b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/zarr.json new file mode 100644 index 0000000000..3e26baa863 --- /dev/null +++ b/testdata/datasource/zarr/zarr_v3/examples/regular_bounded_origin/zarr.json @@ -0,0 +1,43 @@ +{ + "shape": [ + 3, + 4 + ], + "data_type": "uint16", + "chunk_grid": { + "name": "regular_bounded", + "configuration": { + "chunk_shape": [ + 2, + 3 + ], + "origin": [ + 1, + -1 + ], + "lower_bound": [ + -1, + 0 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "attributes": {}, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py b/testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py new file mode 100644 index 0000000000..19fd5ba1fe --- /dev/null +++ b/testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py @@ -0,0 +1,35 @@ +"""Writes `examples/regular_bounded`: a 6 x 7 array grown in front by 3 rows and 2 columns. + +Requires a zarr-python with `Array.prepend` (zarr-developers/zarr-python#4516). +""" + +import pathlib +import shutil + +import numpy as np +import zarr + +out = pathlib.Path(__file__).parent / "examples" / "regular_bounded" +shutil.rmtree(out, ignore_errors=True) +z = zarr.create_array( + store=str(out), shape=(6, 7), chunks=(4, 5), dtype="uint16", fill_value=0, compressors=None +) +z[:] = np.arange(42, dtype="uint16").reshape(6, 7) + 100 +z.prepend(np.arange(21, dtype="uint16").reshape(3, 7) + 1, axis=0) +z.prepend(np.full((9, 2), 900, dtype="uint16"), axis=1) + +# `examples/regular_bounded_origin`: chunk 0 starts at (1, -1), the first element at (-1, 0). +import json + +out = pathlib.Path(__file__).parent / "examples" / "regular_bounded_origin" +shutil.rmtree(out, ignore_errors=True) +zarr.create_array( + store=str(out), shape=(3, 4), chunks=(2, 3), dtype="uint16", fill_value=0, compressors=None +) +meta = json.loads((out / "zarr.json").read_text()) +meta["chunk_grid"] = { + "name": "regular_bounded", + "configuration": {"chunk_shape": [2, 3], "origin": [1, -1], "lower_bound": [-1, 0]}, +} +(out / "zarr.json").write_text(json.dumps(meta, indent=2)) +zarr.open_array(str(out), mode="r+")[:] = np.arange(12, dtype="uint16").reshape(3, 4) + 1 diff --git a/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_offset_grown.snapshot b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_regular_bounded.snapshot similarity index 90% rename from tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_offset_grown.snapshot rename to tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_regular_bounded.snapshot index 5438209024..47560b876f 100644 --- a/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_offset_grown.snapshot +++ b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_regular_bounded.snapshot @@ -1,14 +1,14 @@ { - "canonicalUrl": "http://localhost:*/datasource/zarr/zarr_v3/examples/offset_grown/|zarr3:", + "canonicalUrl": "http://localhost:*/datasource/zarr/zarr_v3/examples/regular_bounded/|zarr3:", "modelTransform": { "inputSpace": { "bounds": { "lowerBounds": Float64Array [ - 0, - 0, + -3, + -2, ], "upperBounds": Float64Array [ - 10, + 6, 7, ], "voxelCenterAtIntegerCoordinates": [ @@ -48,7 +48,7 @@ { "chunkSource": { "parameters": { - "url": "http://localhost:*/datasource/zarr/zarr_v3/examples/offset_grown/", + "url": "http://localhost:*/datasource/zarr/zarr_v3/examples/regular_bounded/", }, "spec": { "baseVoxelOffset": [ @@ -62,12 +62,12 @@ "compressedSegmentationBlockSize": undefined, "dataType": "UINT16", "lowerVoxelBound": [ - 0, - 0, + -2, + -3, ], "upperVoxelBound": [ 7, - 10, + 6, ], }, }, @@ -104,11 +104,11 @@ "description": "Data Bounds", "id": "data-bounds", "pointA": [ - 0, - 0, + -3, + -2, ], "pointB": [ - 10, + 6, 7, ], "type": "axis_aligned_bounding_box", diff --git a/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_regular_bounded_origin.snapshot b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_regular_bounded_origin.snapshot new file mode 100644 index 0000000000..9882085e22 --- /dev/null +++ b/tests/datasource/metadata_snapshots/zarr/zarr_v3_examples_regular_bounded_origin.snapshot @@ -0,0 +1,120 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/zarr/zarr_v3/examples/regular_bounded_origin/|zarr3:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -1, + 0, + ], + "upperBounds": Float64Array [ + 2, + 4, + ], + "voxelCenterAtIntegerCoordinates": [ + false, + false, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "dim_0", + "dim_1", + ], + "scales": Float64Array [ + 1, + 1, + ], + "units": [ + "", + "", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/zarr_v3/examples/regular_bounded_origin/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 3, + 2, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + 1, + -2, + ], + "upperVoxelBound": [ + 5, + 1, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 1, + 1, + ], + [ + 1, + 0, + -1, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + -1, + 0, + ], + "pointB": [ + 2, + 4, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/zarr.browser_test.ts b/tests/datasource/zarr.browser_test.ts index ec76b3733e..501be15109 100644 --- a/tests/datasource/zarr.browser_test.ts +++ b/tests/datasource/zarr.browser_test.ts @@ -24,7 +24,8 @@ import { datasourceMetadataSnapshotTests } from "#tests/datasource/metadata_snap datasourceMetadataSnapshotTests("zarr", [ "zarr_v3/examples/single_res", "zarr_v3/examples/transpose", - "zarr_v3/examples/offset_grown", + "zarr_v3/examples/regular_bounded", + "zarr_v3/examples/regular_bounded_origin", "ome_zarr/simple_0.4", "ome_zarr/simple_0.5", "ome_zarr/simple_0.5.zip", From f1ffd2e2a5e792a04d6dd60146041871ad0db3bc Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Mon, 19 Oct 2026 16:46:15 +0000 Subject: [PATCH 5/6] zarr: anchor an OME level's translation at its first element A regular_bounded array inside an OME multiscale group was placed with `translation` at coordinate 0, which displaced it by lower_bound * scale. The chunk space now maps to an index space that starts at the first element, as OME-NGFF defines it; a bare array keeps its own coordinates through the scale transform. Adds the regular_bounded_0.5 fixture, simple_0.5 grown in front at both levels, whose per-level transforms match the ungrown group. --- src/datasource/zarr/README.md | 4 +- src/datasource/zarr/frontend.ts | 18 +- .../ome_zarr/generate_regular_bounded_0.5.py | 82 +++++++++ .../regular_bounded_0.5/scale0/image/c/-1/-1 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/-1/0 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/-1/1 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/0/-1 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/0/0 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/0/1 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/1/-1 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/1/0 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale0/image/c/1/1 | Bin 0 -> 50 bytes .../scale0/image/zarr.json | 47 +++++ .../regular_bounded_0.5/scale0/zarr.json | 5 + .../regular_bounded_0.5/scale1/image/c/-1/-1 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale1/image/c/-1/0 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale1/image/c/0/-1 | Bin 0 -> 50 bytes .../regular_bounded_0.5/scale1/image/c/0/0 | Bin 0 -> 50 bytes .../scale1/image/zarr.json | 47 +++++ .../regular_bounded_0.5/scale1/zarr.json | 5 + .../ome_zarr/regular_bounded_0.5/zarr.json | 66 +++++++ .../ome_zarr_regular_bounded_0.5.snapshot | 164 ++++++++++++++++++ tests/datasource/zarr.browser_test.ts | 1 + 23 files changed, 430 insertions(+), 9 deletions(-) create mode 100644 testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/-1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/0 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/0/-1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/0/0 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/0/1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/-1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/0 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/zarr.json create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/zarr.json create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/-1/-1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/-1/0 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/0/-1 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/0/0 create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/zarr.json create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/zarr.json create mode 100644 testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/zarr.json create mode 100644 tests/datasource/metadata_snapshots/zarr/ome_zarr_regular_bounded_0.5.snapshot diff --git a/src/datasource/zarr/README.md b/src/datasource/zarr/README.md index c3874833fd..ae75ddd274 100644 --- a/src/datasource/zarr/README.md +++ b/src/datasource/zarr/README.md @@ -58,7 +58,9 @@ Supported chunk grids: - regular - regular_bounded (a regular grid with an `origin` and a `lower_bound`, under which chunk keys - may be negative and the first chunk may be partly filled) + may be negative and the first chunk may be partly filled). A bare array is displayed in its own + coordinates, starting at `lower_bound`. Inside an OME-Zarr multiscale group each level's + `translation` refers to its first element, as OME-NGFF defines the array index space. Supported codecs: diff --git a/src/datasource/zarr/frontend.ts b/src/datasource/zarr/frontend.ts index fd9342c51b..b244835a94 100644 --- a/src/datasource/zarr/frontend.ts +++ b/src/datasource/zarr/frontend.ts @@ -127,7 +127,8 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou const permutedUpperBound = new Float32Array(rank); const orderTransform = new Float32Array((rank + 1) ** 2); orderTransform[(rank + 1) ** 2 - 1] = 1; - // Chunk coordinates are relative to the chunk grid origin. + // Chunk coordinates are relative to the chunk grid origin; the index + // space the scale transform maps starts at the first element. for (let i = 0; i < rank; ++i) { const decodedDim = physicalToLogicalDimension[rank - 1 - i]; permutedChunkShape[i] = readChunkShape[decodedDim]; @@ -135,7 +136,8 @@ export class MultiscaleVolumeChunkSource extends GenericMultiscaleVolumeChunkSou lowerBound[decodedDim] - chunkGridOrigin[decodedDim]; permutedUpperBound[i] = permutedLowerBound[i] + shape[decodedDim]; orderTransform[decodedDim + i * (rank + 1)] = 1; - orderTransform[(rank + 1) * rank + i] = chunkGridOrigin[i]; + orderTransform[(rank + 1) * rank + i] = + chunkGridOrigin[i] - lowerBound[i]; } const transform = new Float32Array((rank + 1) ** 2); matrix.multiply( @@ -289,18 +291,20 @@ function getMultiscaleInfoForSingleArray( }), ], }); - const transform = matrix.createIdentity(Float64Array, metadata.rank + 1); return { coordinateSpace: modelSpace, dataType: metadata.dataType, scales: [ { url, - transform, + transform: matrix.createHomogeneousTranslationMatrix( + Float64Array, + metadata.lowerBound, + ), metadata, }, ], - baseTransform: transform, + baseTransform: matrix.createIdentity(Float64Array, metadata.rank + 1), }; } @@ -356,9 +360,7 @@ async function resolveOmeMultiscale( const baseScale = multiscale.scales[0]; const baseZarrMetadata = scaleZarrMetadata[0]; for (let i = 0; i < rank; ++i) { - const lower = (lowerBounds[i] = - baseScale.transform[(rank + 1) * rank + i] + - baseZarrMetadata.lowerBound[i]); + const lower = (lowerBounds[i] = baseScale.transform[(rank + 1) * rank + i]); upperBounds[i] = lower + baseZarrMetadata.shape[i]; } const boundingBox = makeIdentityTransformedBoundingBox({ diff --git a/testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py b/testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py new file mode 100644 index 0000000000..2a8a9157fb --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py @@ -0,0 +1,82 @@ +"""Writes `regular_bounded_0.5`: the `simple_0.5` image grown in front by 3 rows and 2 columns. + +Requires a zarr-python with `Array.prepend` (zarr-developers/zarr-python#4516). Both levels are +grown with `prepend`, and each level's `translation` moves to its new first element, as OME-NGFF +defines the array index space, so the original elements keep their physical positions. +""" + +import json +import pathlib +import shutil + +import numpy as np +import zarr + +root = pathlib.Path(__file__).parent / "regular_bounded_0.5" +shutil.rmtree(root, ignore_errors=True) +root.mkdir() + + +def level(path, data): + a = zarr.create_array( + store=str(root / path), + shape=data.shape, + chunks=(5, 5), + dtype="uint16", + fill_value=0, + compressors=None, + dimension_names=["y", "x"], + ) + a[:] = data + return a + + +# Level 0: 10 x 10 at 4 x 30 nm, then rows [-3, 0) and columns [-2, 0) in front. +s0 = level("scale0/image", np.arange(100, dtype="uint16").reshape(10, 10)) +s0.prepend(np.arange(30, dtype="uint16").reshape(3, 10) + 1000, axis=0) +s0.prepend(np.arange(26, dtype="uint16").reshape(13, 2) + 2000, axis=1) + +# Level 1: 2 x 2 bins on even level-0 coordinates: rows [-4, 10), columns [-2, 10). +grown = np.zeros((14, 12), dtype="float64") +grown[1:] = s0[:] +binned = grown.reshape(7, 2, 6, 2).mean(axis=(1, 3)).astype("uint16") +s1 = level("scale1/image", binned[2:, 1:]) +s1.prepend(binned[:2, 1:], axis=0) +s1.prepend(binned[:, :1], axis=1) + +# Translations are voxel centres of each level's first element: +# level 0 row -3 -> 4 * -3 = -12 nm, column -2 -> 30 * -2 = -60 nm; +# level 1 rows {-4, -3} -> -14 nm, columns {-2, -1} -> -45 nm. +ome = { + "version": "0.5", + "multiscales": [ + { + "name": "image", + "axes": [ + {"name": "y", "type": "space", "unit": "nanometer"}, + {"name": "x", "type": "space", "unit": "nanometer"}, + ], + "datasets": [ + { + "path": "scale0/image", + "coordinateTransformations": [ + {"type": "scale", "scale": [4, 30]}, + {"type": "translation", "translation": [-12, -60]}, + ], + }, + { + "path": "scale1/image", + "coordinateTransformations": [ + {"type": "scale", "scale": [8, 60]}, + {"type": "translation", "translation": [-14, -45]}, + ], + }, + ], + } + ], +} +for group in (root, root / "scale0", root / "scale1"): + attrs = {"ome": ome} if group == root else {} + (group / "zarr.json").write_text( + json.dumps({"zarr_format": 3, "node_type": "group", "attributes": attrs}, indent=2) + ) diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/-1 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/-1 new file mode 100644 index 0000000000000000000000000000000000000000..8e18d6279681afc2afb6394d7254ee3c395b66da GIT binary patch literal 50 acmZQzzy&U_Uu1{KU1GlsWnE#v$_@ZQ`UM&Q literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/0 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/0 new file mode 100644 index 0000000000000000000000000000000000000000..e695834cc95ce70bdd910ae4fdd2f4a636dfb574 GIT binary patch literal 50 qcmZQzzyvRtUoyX9e$D)b`4jVJ<}b`&nZGgrVgAefkNH0{0}BA*at(U` literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/1 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/-1/1 new file mode 100644 index 0000000000000000000000000000000000000000..9afbbbe5d76dd0f5f2b8148aa83620ed34282e1a GIT binary patch literal 50 ncmZX>$pHWm0KhP`_Mg=Ca(xUS3E@|$G&+OHVsp4elc?DT$~p%?02E8d+hhw0m$$O&;S4c literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/0/0 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..ea25fa73da78ea47fcb9b0fe54816e874305b391 GIT binary patch literal 50 wcmV~$w*i14006KkObSf43DL>+|9vKOdOUYf<~*;8;mB=ELp5JyX|*1`us4|0;B){ literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/-1 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/-1 new file mode 100644 index 0000000000000000000000000000000000000000..34c7713357fc1ce8ce0dc10d678b2b348c207128 GIT binary patch literal 50 kcmZQz0D%YW581&ChDYp=p{ytDPob=5?9ZXB7wj+D0nzLUGynhq literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/0 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/0 new file mode 100644 index 0000000000000000000000000000000000000000..0c3d4f045f6bfbfe8e593ed135272a83efaa36a6 GIT binary patch literal 50 ycmV~$w*i14006Mugk@53!ZM*S+5SJnBxtlcy}@V_&634xv&#;rqPpB3ug?#o76%Cc literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/1 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/c/1/1 new file mode 100644 index 0000000000000000000000000000000000000000..3f52816fe0632f3e94a0574b5bd605c656088083 GIT binary patch literal 50 ycmV~$w*i14006Ljg=JE4!ZM*S+5Z2*Bxtlcy}@V_&634xv&#;rqPpB3ug?#!$Onc1 literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/zarr.json b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/zarr.json new file mode 100644 index 0000000000..342ecd9bba --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 13, + 12 + ], + "data_type": "uint16", + "chunk_grid": { + "name": "regular_bounded", + "configuration": { + "chunk_shape": [ + 5, + 5 + ], + "origin": [ + 0, + 0 + ], + "lower_bound": [ + -3, + -2 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "attributes": {}, + "dimension_names": [ + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/zarr.json b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/zarr.json new file mode 100644 index 0000000000..051870a8e5 --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/zarr.json @@ -0,0 +1,5 @@ +{ + "zarr_format": 3, + "node_type": "group", + "attributes": {} +} \ No newline at end of file diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/-1/-1 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/-1/-1 new file mode 100644 index 0000000000000000000000000000000000000000..47883e7f4dfe0ef732f00195aab863b707bc9bef GIT binary patch literal 50 RcmZQzAO^f(hN-&D4gd_`0mc9T literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/-1/0 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/-1/0 new file mode 100644 index 0000000000000000000000000000000000000000..e4e460c67394461e92fbdaf919b1ba98941547fd GIT binary patch literal 50 gcmZQzzz4oCer5c|_?__w<9Ftt%)gocGXG}=0BY6=$^ZZW literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/0/-1 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/0/-1 new file mode 100644 index 0000000000000000000000000000000000000000..4f5035a6b3a78cdf6a2a6102d5fdb51ade5153fa GIT binary patch literal 50 acmZQzfPm}lPzJ+oIO{%~^%%~2&JF-!Qv`ni literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/0/0 b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..d8f039ace3a151d6d6a03b304e0399ddc4d9a46a GIT binary patch literal 50 zcmZQ&U}xZD;AY@ukYtc%kY$i(P-M_$&}T4YFlI1iaAa_1aAk03@MH*O2xo|7h-Qdo F0030P1j+ya literal 0 HcmV?d00001 diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/zarr.json b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/zarr.json new file mode 100644 index 0000000000..c70682f1d4 --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/zarr.json @@ -0,0 +1,47 @@ +{ + "shape": [ + 7, + 6 + ], + "data_type": "uint16", + "chunk_grid": { + "name": "regular_bounded", + "configuration": { + "chunk_shape": [ + 5, + 5 + ], + "origin": [ + 0, + 0 + ], + "lower_bound": [ + -2, + -1 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "attributes": {}, + "dimension_names": [ + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/zarr.json b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/zarr.json new file mode 100644 index 0000000000..051870a8e5 --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/zarr.json @@ -0,0 +1,5 @@ +{ + "zarr_format": 3, + "node_type": "group", + "attributes": {} +} \ No newline at end of file diff --git a/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/zarr.json b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/zarr.json new file mode 100644 index 0000000000..3d1620988b --- /dev/null +++ b/testdata/datasource/zarr/ome_zarr/regular_bounded_0.5/zarr.json @@ -0,0 +1,66 @@ +{ + "zarr_format": 3, + "node_type": "group", + "attributes": { + "ome": { + "version": "0.5", + "multiscales": [ + { + "name": "image", + "axes": [ + { + "name": "y", + "type": "space", + "unit": "nanometer" + }, + { + "name": "x", + "type": "space", + "unit": "nanometer" + } + ], + "datasets": [ + { + "path": "scale0/image", + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 4, + 30 + ] + }, + { + "type": "translation", + "translation": [ + -12, + -60 + ] + } + ] + }, + { + "path": "scale1/image", + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 8, + 60 + ] + }, + { + "type": "translation", + "translation": [ + -14, + -45 + ] + } + ] + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/datasource/metadata_snapshots/zarr/ome_zarr_regular_bounded_0.5.snapshot b/tests/datasource/metadata_snapshots/zarr/ome_zarr_regular_bounded_0.5.snapshot new file mode 100644 index 0000000000..5f413f1c74 --- /dev/null +++ b/tests/datasource/metadata_snapshots/zarr/ome_zarr_regular_bounded_0.5.snapshot @@ -0,0 +1,164 @@ +{ + "canonicalUrl": "http://localhost:*/datasource/zarr/ome_zarr/regular_bounded_0.5/|zarr3:", + "modelTransform": { + "inputSpace": { + "bounds": { + "lowerBounds": Float64Array [ + -3.5, + -2.5, + ], + "upperBounds": Float64Array [ + 9.5, + 9.5, + ], + "voxelCenterAtIntegerCoordinates": [ + true, + true, + ], + }, + "coordinateArrays": [ + , + , + ], + "names": [ + "y", + "x", + ], + "scales": Float64Array [ + 4e-9, + 3.0000000000000004e-8, + ], + "units": [ + "m", + "m", + ], + "valid": true, + }, + }, + "subsources": [ + { + "default": true, + "id": "default", + "subsource": { + "volume": { + "dataType": "UINT16", + "rank": 2, + "sources": [ + [ + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/regular_bounded_0.5/scale0/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 5, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + -2, + -3, + ], + "upperVoxelBound": [ + 10, + 10, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 1, + -0.5, + ], + [ + 1, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + { + "chunkSource": { + "parameters": { + "url": "http://localhost:*/datasource/zarr/ome_zarr/regular_bounded_0.5/scale1/image/", + }, + "spec": { + "baseVoxelOffset": [ + 0, + 0, + ], + "chunkDataSize": [ + 5, + 5, + ], + "compressedSegmentationBlockSize": undefined, + "dataType": "UINT16", + "lowerVoxelBound": [ + -1, + -2, + ], + "upperVoxelBound": [ + 5, + 5, + ], + }, + }, + "chunkToMultiscaleTransform": [ + [ + 0, + 2, + -0.5, + ], + [ + 2, + 0, + -0.5, + ], + [ + 0, + 0, + 1, + ], + ], + }, + ], + ], + "volumeType": "IMAGE", + }, + }, + }, + { + "default": true, + "id": "bounds", + "subsource": { + "staticAnnotations": [ + { + "description": "Data Bounds", + "id": "data-bounds", + "pointA": [ + -3.5, + -2.5, + ], + "pointB": [ + 9.5, + 9.5, + ], + "type": "axis_aligned_bounding_box", + }, + ], + }, + }, + ], +} \ No newline at end of file diff --git a/tests/datasource/zarr.browser_test.ts b/tests/datasource/zarr.browser_test.ts index 501be15109..810bd7e112 100644 --- a/tests/datasource/zarr.browser_test.ts +++ b/tests/datasource/zarr.browser_test.ts @@ -30,6 +30,7 @@ datasourceMetadataSnapshotTests("zarr", [ "ome_zarr/simple_0.5", "ome_zarr/simple_0.5.zip", "ome_zarr/simple_0.5.ocdbt", + "ome_zarr/regular_bounded_0.5", ]); datasourceMetadataSnapshotTests( From 0250eca9b5bd431eb3403c3603a42da4f10147be Mon Sep 17 00:00:00 2001 From: rhoadesScholar Date: Mon, 19 Oct 2026 21:47:49 +0000 Subject: [PATCH 6/6] zarr: ruff-clean fixture generators Moves a mid-file import to the top (E402) and applies ruff format to the two zarr fixture generators. --- .../ome_zarr/generate_regular_bounded_0.5.py | 4 +++- .../zarr/zarr_v3/generate_regular_bounded.py | 20 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py b/testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py index 2a8a9157fb..345a64c180 100644 --- a/testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py +++ b/testdata/datasource/zarr/ome_zarr/generate_regular_bounded_0.5.py @@ -78,5 +78,7 @@ def level(path, data): for group in (root, root / "scale0", root / "scale1"): attrs = {"ome": ome} if group == root else {} (group / "zarr.json").write_text( - json.dumps({"zarr_format": 3, "node_type": "group", "attributes": attrs}, indent=2) + json.dumps( + {"zarr_format": 3, "node_type": "group", "attributes": attrs}, indent=2 + ) ) diff --git a/testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py b/testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py index 19fd5ba1fe..51dd1d18e9 100644 --- a/testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py +++ b/testdata/datasource/zarr/zarr_v3/generate_regular_bounded.py @@ -3,6 +3,7 @@ Requires a zarr-python with `Array.prepend` (zarr-developers/zarr-python#4516). """ +import json import pathlib import shutil @@ -12,19 +13,28 @@ out = pathlib.Path(__file__).parent / "examples" / "regular_bounded" shutil.rmtree(out, ignore_errors=True) z = zarr.create_array( - store=str(out), shape=(6, 7), chunks=(4, 5), dtype="uint16", fill_value=0, compressors=None + store=str(out), + shape=(6, 7), + chunks=(4, 5), + dtype="uint16", + fill_value=0, + compressors=None, ) z[:] = np.arange(42, dtype="uint16").reshape(6, 7) + 100 z.prepend(np.arange(21, dtype="uint16").reshape(3, 7) + 1, axis=0) z.prepend(np.full((9, 2), 900, dtype="uint16"), axis=1) # `examples/regular_bounded_origin`: chunk 0 starts at (1, -1), the first element at (-1, 0). -import json out = pathlib.Path(__file__).parent / "examples" / "regular_bounded_origin" shutil.rmtree(out, ignore_errors=True) zarr.create_array( - store=str(out), shape=(3, 4), chunks=(2, 3), dtype="uint16", fill_value=0, compressors=None + store=str(out), + shape=(3, 4), + chunks=(2, 3), + dtype="uint16", + fill_value=0, + compressors=None, ) meta = json.loads((out / "zarr.json").read_text()) meta["chunk_grid"] = { @@ -32,4 +42,6 @@ "configuration": {"chunk_shape": [2, 3], "origin": [1, -1], "lower_bound": [-1, 0]}, } (out / "zarr.json").write_text(json.dumps(meta, indent=2)) -zarr.open_array(str(out), mode="r+")[:] = np.arange(12, dtype="uint16").reshape(3, 4) + 1 +zarr.open_array(str(out), mode="r+")[:] = ( + np.arange(12, dtype="uint16").reshape(3, 4) + 1 +)