From e9b91843201f492d6422b012b35ab2449d3fe3a0 Mon Sep 17 00:00:00 2001 From: Trevor Manz Date: Thu, 6 Mar 2025 12:01:43 -0500 Subject: [PATCH] Apply OME coordinate transformations --- .../LayerController/AddChannelButton.tsx | 4 +- src/components/Viewer.tsx | 67 +++++------ src/io.ts | 24 ++-- src/{gridLayer.ts => layers/grid-layer.ts} | 42 +++---- src/layers/viv-layers.ts | 54 +++++++++ src/ome.ts | 8 +- src/state.ts | 65 ++++++---- src/types.ts | 17 ++- src/utils.ts | 111 +++++++++++++++--- 9 files changed, 275 insertions(+), 117 deletions(-) rename src/{gridLayer.ts => layers/grid-layer.ts} (86%) create mode 100644 src/layers/viv-layers.ts diff --git a/src/components/LayerController/AddChannelButton.tsx b/src/components/LayerController/AddChannelButton.tsx index 8e3df001..f4801fc2 100644 --- a/src/components/LayerController/AddChannelButton.tsx +++ b/src/components/LayerController/AddChannelButton.tsx @@ -4,7 +4,7 @@ import React, { useState } from "react"; import type { ChangeEvent, MouseEvent } from "react"; import { useLayerState, useSourceData } from "../../hooks"; -import { MAX_CHANNELS, calcDataRange, hexToRGB } from "../../utils"; +import { MAX_CHANNELS, calcDataRange, hexToRGB, resolveLoaderFromLayerProps } from "../../utils"; function AddChannelButton() { const [source, setSource] = useSourceData(); @@ -32,7 +32,7 @@ function AddChannelButton() { if (source.contrast_limits[channelIndex]) { lim = source.contrast_limits[channelIndex] as [number, number]; } else { - const { loader } = layer.layerProps; + const loader = resolveLoaderFromLayerProps(layer.layerProps); const lowres = Array.isArray(loader) ? loader[loader.length - 1] : loader; lim = await calcDataRange(lowres, channelSelection); // Update source data with newly calculated limit diff --git a/src/components/Viewer.tsx b/src/components/Viewer.tsx index d00efb76..e0334b9f 100644 --- a/src/components/Viewer.tsx +++ b/src/components/Viewer.tsx @@ -2,44 +2,31 @@ import DeckGL from "deck.gl"; import { OrthographicView } from "deck.gl"; import { useAtomValue } from "jotai"; import * as React from "react"; - -import type { DeckGLRef, Layer, LayerProps, OrthographicViewState } from "deck.gl"; -import type { ZarrPixelSource } from "../ZarrPixelSource"; import { useViewState } from "../hooks"; import { layerAtoms } from "../state"; -import { fitBounds, isInterleaved } from "../utils"; - -type Data = { loader: ZarrPixelSource; rows: number; columns: number }; -type VizarrLayer = Layer; +import { fitImageToViewport, isGridLayerProps, isInterleaved, resolveLoaderFromLayerProps } from "../utils"; -function getLayerSize(props: Data) { - const { loader } = props; - const [base, maxZoom] = Array.isArray(loader) ? [loader[0], loader.length] : [loader, 0]; - const interleaved = isInterleaved(base.shape); - let [height, width] = base.shape.slice(interleaved ? -3 : -2); - if ("loaders" in props && props.rows && props.columns) { - // TODO: Don't hardcode spacer size. Probably best to inspect the deck.gl Layers rather than - // the Layer Props. - const spacer = 5; - height = (height + spacer) * props.rows; - width = (width + spacer) * props.columns; - } - return { height, width, maxZoom }; -} +import type { DeckGLRef, OrthographicViewState } from "deck.gl"; +import type { VizarrLayer } from "../state"; -function WrappedViewStateDeck(props: { layers: Array }) { +export default function Viewer() { const deckRef = React.useRef(null); const [viewState, setViewState] = useViewState(); - const firstLayerProps = props.layers[0]?.props; + const layers = useAtomValue(layerAtoms); + const firstLayer = layers[0]; // If viewState hasn't been updated, use the first loader to guess viewState // TODO: There is probably a better place / way to set the intital view and this is a hack. - if (deckRef.current?.deck && !viewState && firstLayerProps?.loader) { + if (deckRef.current?.deck && !viewState && firstLayer) { const { deck } = deckRef.current; - const { width, height, maxZoom } = getLayerSize(firstLayerProps); - const padding = deck.width < 400 ? 10 : deck.width < 600 ? 30 : 50; // Adjust depending on viewport width. - const bounds = fitBounds([width, height], [deck.width, deck.height], maxZoom, padding); - setViewState(bounds); + setViewState( + fitImageToViewport({ + image: getLayerSize(firstLayer), + viewport: deck, + padding: deck.width < 400 ? 10 : deck.width < 600 ? 30 : 50, // Adjust depending on viewport width. + matrix: firstLayer.props.modelMatrix, + }), + ); } // Enables screenshots of the canvas: https://github.com/visgl/deck.gl/issues/2200 @@ -50,7 +37,7 @@ function WrappedViewStateDeck(props: { layers: Array }) { return ( // @ts-expect-error - deck doesn't know this should be ok @@ -62,13 +49,17 @@ function WrappedViewStateDeck(props: { layers: Array }) { ); } -function Viewer() { - const layerConstructors = useAtomValue(layerAtoms); - // @ts-expect-error - Viv types are giving up an issue - const layers: Array = layerConstructors.map((layer) => { - return !layer.on ? null : new layer.Layer(layer.layerProps); - }); - return ; +function getLayerSize({ props }: VizarrLayer) { + const loader = resolveLoaderFromLayerProps(props); + const [baseResolution, maxZoom] = Array.isArray(loader) ? [loader[0], loader.length] : [loader, 0]; + const interleaved = isInterleaved(baseResolution.shape); + let [height, width] = baseResolution.shape.slice(interleaved ? -3 : -2); + if (isGridLayerProps(props)) { + // TODO: Don't hardcode spacer size. Probably best to inspect the deck.gl Layers rather than + // the Layer Props. + const spacer = 5; + height = (height + spacer) * props.rows; + width = (width + spacer) * props.columns; + } + return { height, width, maxZoom }; } - -export default Viewer; diff --git a/src/io.ts b/src/io.ts index d545bb30..d0dc52d4 100644 --- a/src/io.ts +++ b/src/io.ts @@ -1,12 +1,11 @@ -import { ImageLayer, MultiscaleImageLayer } from "@hms-dbmi/viv"; import * as zarr from "zarrita"; - import { ZarrPixelSource } from "./ZarrPixelSource"; -import GridLayer from "./gridLayer"; -import { loadOmeroMultiscales, loadPlate, loadWell } from "./ome"; -import type { ImageLayerConfig, LayerState, MultichannelConfig, SingleChannelConfig, SourceData } from "./state"; +import { loadOmeMultiscales, loadPlate, loadWell } from "./ome"; import * as utils from "./utils"; +import type { BaseLayerProps } from "./layers/viv-layers"; +import type { ImageLayerConfig, LayerState, MultichannelConfig, SingleChannelConfig, SourceData } from "./state"; + async function loadSingleChannel(config: SingleChannelConfig, data: Array): Promise { const { color, contrast_limits, visibility, name, colormap = "", opacity = 1 } = config; const lowres = data[data.length - 1]; @@ -89,8 +88,8 @@ export async function createSourceData(config: ImageLayerConfig): Promise { +class GridLayer extends CompositeLayer { + static layerName = "VizarrGridLayer"; + static defaultProps = { + // @ts-expect-error - XRLayer props are not typed + ...XRLayer.defaultProps, + // Special grid props + loaders: { type: "array", value: [], compare: true }, + spacer: { type: "number", value: 5, compare: true }, + rows: { type: "number", value: 0, compare: true }, + columns: { type: "number", value: 0, compare: true }, + concurrency: { type: "number", value: 10, compare: false }, // set concurrency for queue + text: { type: "boolean", value: false, compare: true }, + // Deck.gl + onClick: { type: "function", value: null, compare: true }, + onHover: { type: "function", value: null, compare: true }, + }; + get #state(): SharedLayerState { // @ts-expect-error - typed as any by deck return this.state; @@ -210,5 +211,4 @@ export default class GridLayer extends CompositeLayer; + colors: [r: number, g: number, b: number][]; + channelsVisible: Array; + opacity: number; + colormap: string; // TODO: more precise + selections: number[][]; + modelMatrix: Matrix4; + contrastLimitsRange: [min: number, max: number][]; + onClick?: (e: Record) => void; +} + +export interface MultiscaleImageLayerProps extends BaseLayerProps { + loader: Array; +} + +// @ts-expect-error - Viv does faithfully implement the Layer interface, just not captured in the types +export class MultiscaleImageLayer + extends BaseMultiscaleImageLayer + implements Layer +{ + static layerName = "VizarMultiscaleImageLayer"; + // biome-ignore lint/complexity/noUselessConstructor: Necessary for TypeScript to get the types + constructor(props: MultiscaleImageLayerProps) { + // @ts-expect-error - Viv's types are not correct + super(props); + } +} + +export interface ImageLayerProps extends BaseLayerProps { + loader: ZarrPixelSource; +} + +// @ts-expect-error - Viv does faithfully implement the Layer interface, just not captured in the types +export class ImageLayer extends BaseImageLayer implements Layer { + static layerName = "VizarrImageLayer"; + // biome-ignore lint/complexity/noUselessConstructor: Necessary for TypeScript to get the types + constructor(props: ImageLayerProps) { + // @ts-expect-error - Viv's types are not correct + super(props); + } +} diff --git a/src/ome.ts b/src/ome.ts index 7e1d221c..4d5396b7 100644 --- a/src/ome.ts +++ b/src/ome.ts @@ -76,7 +76,7 @@ export async function loadWell( }); let meta: Meta; - if (utils.isOmeroMultiscales(imgAttrs)) { + if (utils.isOmeMultiscales(imgAttrs)) { meta = parseOmeroMeta(imgAttrs.omero, axes); } else { meta = await defaultMeta(loaders[0].loader, axis_labels); @@ -242,7 +242,7 @@ export async function loadPlate( return sourceData; } -export async function loadOmeroMultiscales( +export async function loadOmeMultiscales( config: ImageLayerConfig, grp: zarr.Group, attrs: { multiscales: Ome.Multiscale[]; omero: Ome.Omero }, @@ -258,7 +258,9 @@ export async function loadOmeroMultiscales( return { loader: loader, axis_labels, - model_matrix: utils.parseMatrix(config.model_matrix), + model_matrix: config.model_matrix + ? utils.parseMatrix(config.model_matrix) + : utils.coordinateTransformationsToMatrix(attrs.multiscales), defaults: { selection: meta.defaultSelection, colormap, diff --git a/src/state.ts b/src/state.ts index 3d492b4b..caf23587 100644 --- a/src/state.ts +++ b/src/state.ts @@ -1,13 +1,22 @@ -import type { ImageLayer, MultiscaleImageLayer } from "@hms-dbmi/viv"; import { atom } from "jotai"; import { atomFamily, splitAtom, waitForAll } from "jotai/utils"; -import type { Matrix4 } from "math.gl"; +import { RedirectError, rethrowUnless } from "./utils"; +import type { Layer } from "deck.gl"; +import type { PrimitiveAtom } from "jotai"; +import type { AtomFamily } from "jotai/vanilla/utils/atomFamily"; +import type { Matrix4 } from "math.gl"; import type * as zarr from "zarrita"; import type { ZarrPixelSource } from "./ZarrPixelSource"; -import type { default as GridLayer, GridLayerProps, GridLoader } from "./gridLayer"; import { initLayerStateFromSource } from "./io"; -import { RedirectError, rethrowUnless } from "./utils"; + +import { GridLayer, type GridLayerProps, type GridLoader } from "./layers/grid-layer"; +import { + ImageLayer, + type ImageLayerProps, + MultiscaleImageLayer, + type MultiscaleImageLayerProps, +} from "./layers/viv-layers"; export interface ViewState { zoom: number; @@ -83,25 +92,16 @@ export interface BaseLayerProps { onClick?: (e: OnClickData) => void; } -interface MultiscaleImageLayerProps extends BaseLayerProps { - loader: Array; -} - -interface ImageLayerProps extends BaseLayerProps { - loader: ZarrPixelSource; -} - -type LayerMap = { - image: [typeof ImageLayer, ImageLayerProps]; - multiscale: [typeof MultiscaleImageLayer, MultiscaleImageLayerProps]; - grid: [GridLayer, { loader: ZarrPixelSource | Array } & GridLayerProps]; +type LayerType = "image" | "multiscale" | "grid"; +type LayerPropsMap = { + image: ImageLayerProps; + multiscale: MultiscaleImageLayerProps; + grid: GridLayerProps; }; -// biome-ignore lint/suspicious/noExplicitAny: Need a catch all for layer types -export type LayerCtr = new (...args: Array) => T; -export type LayerState = { - Layer: LayerCtr; - layerProps: LayerMap[T][1]; +export type LayerState = { + kind: T; + layerProps: LayerPropsMap[T]; on: boolean; }; @@ -121,6 +121,7 @@ export const sourceInfoAtom = atom[]>([]); export const addImageAtom = atom(null, async (get, set, config: ImageLayerConfig) => { const { createSourceData } = await import("./io"); const id = Math.random().toString(36).slice(2); + try { const sourceData = await createSourceData(config); const prevSourceInfo = get(sourceInfoAtom); @@ -140,14 +141,28 @@ export const addImageAtom = atom(null, async (get, set, config: ImageLayerConfig export const sourceInfoAtomAtoms = splitAtom(sourceInfoAtom); -export const layerFamilyAtom = atomFamily( +export const layerFamilyAtom: AtomFamily, PrimitiveAtom>> = atomFamily( (param: WithId) => atom({ ...initLayerStateFromSource(param), id: param.id }), (a, b) => a.id === b.id, ); +export type VizarrLayer = Layer | Layer | Layer; + +const LayerConstructors = { + image: ImageLayer, + multiscale: MultiscaleImageLayer, + grid: GridLayer, +} as const; + export const layerAtoms = atom((get) => { const atoms = get(sourceInfoAtomAtoms); - if (atoms.length === 0) return []; - const layers = atoms.map((a) => layerFamilyAtom(get(a))); - return get(waitForAll(layers)); + if (atoms.length === 0) { + return []; + } + const layersState = get(waitForAll(atoms.map((a) => layerFamilyAtom(get(a))))); + return layersState.map((layer) => { + const Layer = LayerConstructors[layer.kind]; + // @ts-expect-error - TS can't resolve that Layer & layerProps bound together + return new Layer(layer.layerProps); + }) as Array; }); diff --git a/src/types.ts b/src/types.ts index 397212f3..9c800b23 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,8 +33,23 @@ declare namespace Ome { type?: string; } + type CoordinateTransformation = + | { + type: "scale"; + scale: Array; + } + | { + type: "translation"; + translation: Array; + }; + + interface Dataset { + path: string; + coordinateTransformations?: Array; + } + interface Multiscale { - datasets: { path: string }[]; + datasets: Array; version?: string; axes?: string[] | Axis[]; } diff --git a/src/utils.ts b/src/utils.ts index 1babeb17..ed64afe1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,6 +2,8 @@ import { Matrix4 } from "math.gl"; import * as zarr from "zarrita"; import type { ZarrPixelSource } from "./ZarrPixelSource"; +import type { GridLayerProps } from "./layers/grid-layer"; +import type { ImageLayerProps, MultiscaleImageLayerProps } from "./layers/viv-layers"; import { lru } from "./lru-store"; import type { ViewState } from "./state"; @@ -219,16 +221,37 @@ export function guessTileSize(arr: zarr.Array) { } // Scales the real image size to the target viewport. -export function fitBounds( - [width, height]: [width: number, height: number], - [targetWidth, targetHeight]: [targetWidth: number, targetHeight: number], - maxZoom: number, - padding: number, -): ViewState { - const scaleX = (targetWidth - padding * 2) / width; - const scaleY = (targetHeight - padding * 2) / height; - const zoom = Math.min(maxZoom, Math.log2(Math.min(scaleX, scaleY))); - return { zoom, target: [width / 2, height / 2] }; +export function fitImageToViewport(options: { + image: { width: number; height: number }; + viewport: { width: number; height: number }; + padding: number; + matrix?: Matrix4; +}): ViewState { + const { image, viewport, padding, matrix = new Matrix4().identity() } = options; + const corners = [ + [0, 0, 0], + [image.width, 0, 0], + [image.width, image.height, 0], + [0, image.height, 0], + ].map((corner) => matrix.transformAsPoint(corner)); + + const minX = Math.min(...corners.map((p) => p[0])); + const maxX = Math.max(...corners.map((p) => p[0])); + const minY = Math.min(...corners.map((p) => p[1])); + const maxY = Math.max(...corners.map((p) => p[1])); + + const availableWidth = viewport.width - 2 * padding; + const availableHeight = viewport.height - 2 * padding; + + return { + zoom: Math.log2( + Math.min( + availableWidth / (maxX - minX), // scaleX + availableHeight / (maxY - minX), // scaleY + ), + ), + target: [(minX + maxX) / 2, (minY + maxY) / 2], + }; } type Array16 = [ @@ -433,10 +456,8 @@ export function isOmeWell(attrs: zarr.Attributes): attrs is { well: Ome.Well } { return "well" in attrs; } -export function isOmeroMultiscales( - attrs: zarr.Attributes, -): attrs is { omero: Ome.Omero; multiscales: Ome.Multiscale[] } { - return "omero" in attrs && "multiscales" in attrs; +export function isOmeMultiscales(attrs: zarr.Attributes): attrs is { omero: Ome.Omero; multiscales: Ome.Multiscale[] } { + return "omero" in attrs && isMultiscales(attrs); } export function isMultiscales(attrs: zarr.Attributes): attrs is { multiscales: Ome.Multiscale[] } { @@ -485,3 +506,65 @@ export function rethrowUnless Er throw error; } } + +export function isGridLayerProps( + props: GridLayerProps | ImageLayerProps | MultiscaleImageLayerProps, +): props is GridLayerProps { + return "loaders" in props && "rows" in props && "columns" in props; +} + +export function resolveLoaderFromLayerProps(layerProps: GridLayerProps | ImageLayerProps | MultiscaleImageLayerProps) { + return isGridLayerProps(layerProps) ? layerProps.loaders[0].loader : layerProps.loader; +} + +/** + * Convert an array of coordinateTransformations objects to a 16-element + * plain JS array using Matrix4 linear algebra transformation functions. + * + * Adapted from Vitessce: https://github.com/vitessce/vitessce/blob/c267ebecab1824dae68d6f2640a6c5ce7250efbb/packages/utils/spatial-utils/src/spatial.js#L403-L524 + * + * @param coordinateTransformations List of objects matching the OME-NGFF v0.4 coordinateTransformations spec. + * @param axes - Axes in OME-NGFF v0.4 format + * + * @returns Array of 16 numbers representing the Matrix4. + */ +export function coordinateTransformationsToMatrix(multiscales: Array) { + let mat = new Matrix4().identity(); + const axes = getNgffAxes(multiscales); + const coordinateTransformations = multiscales[0].datasets[0]?.coordinateTransformations; + const xyzIndices = ["x", "y", "z"].map((name) => + axes.findIndex((axisObj) => axisObj.type === "space" && axisObj.name === name), + ); + + // Apply each transformation sequentially and in order according to the OME-NGFF v0.4 spec. + // Reference: https://ngff.openmicroscopy.org/0.4/#trafo-md + for (const transform of coordinateTransformations ?? []) { + if (transform.type === "translation") { + const { translation: axisOrderedTranslation } = transform; + if (axisOrderedTranslation.length !== axes.length) { + throw new Error("Length of translation array was expected to match length of axes."); + } + const defaultValue = 0; + // Get the translation values for [x, y, z]. + const xyzTranslation = xyzIndices.map((axisIndex) => + axisIndex >= 0 ? axisOrderedTranslation[axisIndex] : defaultValue, + ); + const nextMat = new Matrix4().translate(xyzTranslation); + mat = mat.multiplyLeft(nextMat); + } + if (transform.type === "scale") { + const { scale: axisOrderedScale } = transform; + // Add in z dimension needed for Matrix4 scale API. + if (axisOrderedScale.length !== axes.length) { + throw new Error("Length of scale array was expected to match length of axes."); + } + const defaultValue = 1; + // Get the scale values for [x, y, z]. + const xyzScale = xyzIndices.map((axisIndex) => (axisIndex >= 0 ? axisOrderedScale[axisIndex] : defaultValue)); + const nextMat = new Matrix4().scale(xyzScale); + mat = mat.multiplyLeft(nextMat); + } + } + + return mat; +}