diff --git a/docs/examples/plot_fixel_workflow.py b/docs/examples/plot_fixel_workflow.py
index 9f72a65..a350201 100644
--- a/docs/examples/plot_fixel_workflow.py
+++ b/docs/examples/plot_fixel_workflow.py
@@ -102,7 +102,7 @@
# --index-file /home/username/myProject/data/FD/index.mif \
# --directions-file /home/username/myProject/data/FD/directions.mif \
# --cohort-file /home/username/myProject/data/cohort_FD.csv \
-# --output-hdf5 /home/username/myProject/data/FD.h5
+# --output /home/username/myProject/data/FD.h5
#
# This produces ``FD.h5`` in ``/home/username/myProject/data``. You can then use
# `ModelArray `_ to run statistical analyses on it.
diff --git a/docs/examples/plot_voxel_workflow.py b/docs/examples/plot_voxel_workflow.py
index c0a810e..f4810c0 100644
--- a/docs/examples/plot_voxel_workflow.py
+++ b/docs/examples/plot_voxel_workflow.py
@@ -107,7 +107,7 @@
# modelarrayio nifti-to-h5 \
# --group-mask-file /home/username/myProject/data/group_mask.nii.gz \
# --cohort-file /home/username/myProject/data/cohort_FA.csv \
-# --output-hdf5 /home/username/myProject/data/FA.h5
+# --output /home/username/myProject/data/FA.h5
#
# This produces ``FA.h5`` in ``/home/username/myProject/data``. You can then use
# `ModelArray `_ to run statistical analyses on it.
diff --git a/src/modelarrayio/cli/cifti_to_h5.py b/src/modelarrayio/cli/cifti_to_h5.py
index 3dabd5c..50af90d 100644
--- a/src/modelarrayio/cli/cifti_to_h5.py
+++ b/src/modelarrayio/cli/cifti_to_h5.py
@@ -12,12 +12,10 @@
from modelarrayio.cli.parser_utils import (
add_backend_arg,
add_cohort_arg,
- add_output_hdf5_arg,
- add_output_tiledb_arg,
+ add_output_arg,
add_s3_workers_arg,
add_scalar_columns_arg,
add_storage_args,
- add_tiledb_storage_args,
)
from modelarrayio.storage import h5_storage, tiledb_storage
from modelarrayio.utils.cifti import (
@@ -34,68 +32,53 @@
def cifti_to_h5(
cohort_file,
backend='hdf5',
- output_hdf5='fixeldb.h5',
- output_tiledb='arraydb.tdb',
+ output='fixelarray.h5',
storage_dtype='float32',
compression='gzip',
compression_level=4,
shuffle=True,
chunk_voxels=0,
target_chunk_mb=2.0,
- tdb_compression='zstd',
- tdb_compression_level=5,
- tdb_shuffle=True,
- tdb_tile_voxels=0,
- tdb_target_tile_mb=2.0,
- tdb_workers=None,
+ workers=None,
scalar_columns=None,
s3_workers=1,
):
- """Load all CIFTI data and write to an HDF5 file with configurable storage.
+ """Load all CIFTI data and write to an HDF5 or TileDB file.
Parameters
----------
cohort_file : :obj:`str`
Path to a csv with demographic info and paths to data
backend : :obj:`str`
- Backend to use for storage
- output_hdf5 : :obj:`str`
- Path to a new .h5 file to be written
- output_tiledb : :obj:`str`
- Path to a new .tdb file to be written
+ Backend to use for storage (``'hdf5'`` or ``'tiledb'``)
+ output : :obj:`str`
+ Output path. For the hdf5 backend, path to an .h5 file;
+ for the tiledb backend, path to a .tdb directory.
storage_dtype : :obj:`str`
Floating type to store values
compression : :obj:`str`
- HDF5 compression filter
+ Compression filter. ``gzip`` works for both backends;
+ ``lzf`` is HDF5-only; ``zstd`` is TileDB-only.
compression_level : :obj:`int`
- Gzip compression level (0-9)
+ Compression level (codec-dependent)
shuffle : :obj:`bool`
- Enable HDF5 shuffle filter
+ Enable shuffle filter
chunk_voxels : :obj:`int`
- Chunk size along the voxel axis
+ Chunk/tile size along the greyordinate axis (0 = auto)
target_chunk_mb : :obj:`float`
- Target chunk size in MiB when auto-computing chunk_voxels
- tdb_compression : :obj:`str`
- TileDB compression filter
- tdb_compression_level : :obj:`int`
- TileDB compression level
- tdb_shuffle : :obj:`bool`
- Enable TileDB shuffle filter
- tdb_tile_voxels : :obj:`int`
- Tile size along the voxel axis
- tdb_target_tile_mb : :obj:`float`
- Target tile size in MiB when auto-computing tdb_tile_voxels
- tdb_workers : :obj:`int`
- Number of workers to use for parallel loading
+ Target chunk/tile size in MiB when auto-computing the spatial axis length
+ workers : :obj:`int`
+ Maximum number of parallel TileDB write workers (``None`` = auto).
+ Has no effect when ``backend='hdf5'``.
scalar_columns : :obj:`list`
List of scalar columns to use
s3_workers : :obj:`int`
- Number of workers to use for parallel loading
+ Number of workers for parallel S3 downloads
Returns
-------
status : :obj:`int`
- Status of the operation. 0 if successful, 1 if failed.
+ 0 if successful, 1 if failed.
"""
cohort_df = pd.read_csv(cohort_file)
cohort_long = _cohort_to_long_dataframe(cohort_df, scalar_columns=scalar_columns)
@@ -108,7 +91,7 @@ def cifti_to_h5(
if backend == 'hdf5':
scalars, last_brain_names = _load_cohort_cifti(cohort_long, s3_workers)
- f = h5py.File(output_hdf5, 'w')
+ f = h5py.File(output, 'w')
greyordinate_table, structure_names = brain_names_to_dataframe(last_brain_names)
greyordinatesh5 = f.create_dataset(
@@ -136,9 +119,9 @@ def cifti_to_h5(
h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name])
f.close()
- return int(not os.path.exists(output_hdf5))
+ return int(not os.path.exists(output))
else:
- os.makedirs(output_tiledb, exist_ok=True)
+ os.makedirs(output, exist_ok=True)
if not scalar_sources:
return 0
@@ -161,26 +144,26 @@ def _process_scalar_job(scalar_name, source_files):
return scalar_name
num_items = rows[0].shape[0]
tiledb_storage.create_empty_scalar_matrix_array(
- output_tiledb,
+ output,
dataset_path,
num_subjects,
num_items,
storage_dtype=storage_dtype,
- compression=tdb_compression,
- compression_level=tdb_compression_level,
- shuffle=tdb_shuffle,
- tile_voxels=tdb_tile_voxels,
- target_tile_mb=tdb_target_tile_mb,
+ compression=compression,
+ compression_level=compression_level,
+ shuffle=shuffle,
+ tile_voxels=chunk_voxels,
+ target_tile_mb=target_chunk_mb,
sources_list=source_files,
)
# write column names array for ModelArray compatibility
- tiledb_storage.write_column_names(output_tiledb, scalar_name, source_files)
- uri = os.path.join(output_tiledb, dataset_path)
+ tiledb_storage.write_column_names(output, scalar_name, source_files)
+ uri = os.path.join(output, dataset_path)
tiledb_storage.write_rows_in_column_stripes(uri, rows)
return scalar_name
scalar_names = list(scalar_sources.keys())
- worker_count = tdb_workers if isinstance(tdb_workers, int) and tdb_workers > 0 else None
+ worker_count = workers if isinstance(workers, int) and workers > 0 else None
if worker_count is None:
cpu_count = os.cpu_count() or 1
worker_count = min(len(scalar_names), max(1, cpu_count))
@@ -207,20 +190,14 @@ def _process_scalar_job(scalar_name, source_files):
def cifti_to_h5_main(
cohort_file,
backend='hdf5',
- output_hdf5='fixelarray.h5',
- output_tiledb='arraydb.tdb',
+ output='fixelarray.h5',
storage_dtype='float32',
compression='gzip',
compression_level=4,
shuffle=True,
chunk_voxels=0,
target_chunk_mb=2.0,
- tdb_compression='zstd',
- tdb_compression_level=5,
- tdb_shuffle=True,
- tdb_tile_voxels=0,
- tdb_target_tile_mb=2.0,
- tdb_workers=None,
+ workers=None,
scalar_columns=None,
s3_workers=1,
log_level='INFO',
@@ -233,20 +210,14 @@ def cifti_to_h5_main(
return cifti_to_h5(
cohort_file=cohort_file,
backend=backend,
- output_hdf5=output_hdf5,
- output_tiledb=output_tiledb,
+ output=output,
storage_dtype=storage_dtype,
compression=compression,
compression_level=compression_level,
shuffle=shuffle,
chunk_voxels=chunk_voxels,
target_chunk_mb=target_chunk_mb,
- tdb_compression=tdb_compression,
- tdb_compression_level=tdb_compression_level,
- tdb_shuffle=tdb_shuffle,
- tdb_tile_voxels=tdb_tile_voxels,
- tdb_target_tile_mb=tdb_target_tile_mb,
- tdb_workers=tdb_workers,
+ workers=workers,
scalar_columns=scalar_columns,
s3_workers=s3_workers,
)
@@ -254,23 +225,22 @@ def cifti_to_h5_main(
def _parse_cifti_to_h5():
parser = argparse.ArgumentParser(
- description='Create a hdf5 file of CIDTI2 dscalar data',
+ description='Create a hdf5 file of CIFTI2 dscalar data',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
add_cohort_arg(parser)
add_scalar_columns_arg(parser)
- add_output_hdf5_arg(parser, default_name='fixelarray.h5')
- add_output_tiledb_arg(parser, default_name='arraydb.tdb')
+ add_output_arg(parser, default_name='fixelarray.h5')
add_backend_arg(parser)
add_storage_args(parser)
- add_tiledb_storage_args(parser)
parser.add_argument(
- '--tdb-workers',
- '--tdb_workers',
+ '--workers',
type=int,
help=(
- 'Maximum number of TileDB write workers. Default 0 (auto, uses CPU count). '
- 'Set to 1 to disable parallel writes.'
+ 'Maximum number of parallel TileDB write workers. '
+ 'Default 0 (auto, uses CPU count). '
+ 'Set to 1 to disable parallel writes. '
+ 'Has no effect when --backend=hdf5.'
),
default=0,
)
diff --git a/src/modelarrayio/cli/mif_to_h5.py b/src/modelarrayio/cli/mif_to_h5.py
index d2f95e8..9a55bb6 100644
--- a/src/modelarrayio/cli/mif_to_h5.py
+++ b/src/modelarrayio/cli/mif_to_h5.py
@@ -14,10 +14,8 @@
_is_file,
add_backend_arg,
add_cohort_arg,
- add_output_hdf5_arg,
- add_output_tiledb_arg,
+ add_output_arg,
add_storage_args,
- add_tiledb_storage_args,
)
from modelarrayio.storage import h5_storage, tiledb_storage
from modelarrayio.utils.fixels import gather_fixels, mif_to_nifti2
@@ -30,21 +28,15 @@ def mif_to_h5(
directions_file,
cohort_file,
backend='hdf5',
- output_hdf5=Path('fixeldb.h5'),
- output_tiledb=Path('arraydb.tdb'),
+ output=Path('fixelarray.h5'),
storage_dtype='float32',
compression='gzip',
compression_level=4,
shuffle=True,
chunk_voxels=0,
target_chunk_mb=2.0,
- tdb_compression='zstd',
- tdb_compression_level=5,
- tdb_shuffle=True,
- tdb_tile_voxels=0,
- tdb_target_tile_mb=2.0,
):
- """Load all fixeldb data and write to an HDF5 file with configurable storage.
+ """Load all fixeldb data and write to an HDF5 or TileDB file.
Parameters
----------
@@ -55,38 +47,28 @@ def mif_to_h5(
cohort_file : :obj:`pathlib.Path`
Path to a csv with demographic info and paths to data
backend : :obj:`str`
- Backend to use for storage
- output_hdf5 : :obj:`pathlib.Path`
- Path to a new .h5 file to be written
- output_tiledb : :obj:`pathlib.Path`
- Path to a new .tdb file to be written
+ Backend to use for storage (``'hdf5'`` or ``'tiledb'``)
+ output : :obj:`pathlib.Path`
+ Output path. For the hdf5 backend, path to an .h5 file;
+ for the tiledb backend, path to a .tdb directory.
storage_dtype : :obj:`str`
Floating type to store values
compression : :obj:`str`
- HDF5 compression filter
+ Compression filter. ``gzip`` works for both backends;
+ ``lzf`` is HDF5-only; ``zstd`` is TileDB-only.
compression_level : :obj:`int`
- Gzip compression level (0-9)
+ Compression level (codec-dependent)
shuffle : :obj:`bool`
- Enable HDF5 shuffle filter
+ Enable shuffle filter
chunk_voxels : :obj:`int`
- Chunk size along the voxel axis
+ Chunk/tile size along the fixel axis (0 = auto)
target_chunk_mb : :obj:`float`
- Target chunk size in MiB when auto-computing chunk_voxels
- tdb_compression : :obj:`str`
- TileDB compression filter
- tdb_compression_level : :obj:`int`
- TileDB compression level
- tdb_shuffle : :obj:`bool`
- Enable TileDB shuffle filter
- tdb_tile_voxels : :obj:`int`
- Tile size along the voxel axis
- tdb_target_tile_mb : :obj:`float`
- Target tile size in MiB when auto-computing tdb_tile_voxels
+ Target chunk/tile size in MiB when auto-computing the spatial axis length
Returns
-------
status : :obj:`int`
- Status of the operation. 0 if successful, 1 if failed.
+ 0 if successful, 1 if failed.
"""
# gather fixel data
fixel_table, voxel_table = gather_fixels(index_file, directions_file)
@@ -98,17 +80,15 @@ def mif_to_h5(
scalars = defaultdict(list)
sources_lists = defaultdict(list)
print('Extracting .mif data...')
- # ix: index of row (start from 0); row: one row of data
for _ix, row in tqdm(cohort_df.iterrows(), total=cohort_df.shape[0]):
scalar_file = row['source_file']
_scalar_img, scalar_data = mif_to_nifti2(scalar_file)
- scalars[row['scalar_name']].append(scalar_data) # append to specific scalar_name
- # append source mif filename to specific scalar_name
+ scalars[row['scalar_name']].append(scalar_data)
sources_lists[row['scalar_name']].append(row['source_file'])
# Write the output
if backend == 'hdf5':
- f = h5py.File(output_hdf5, 'w')
+ f = h5py.File(output, 'w')
fixelsh5 = f.create_dataset(name='fixels', data=fixel_table.to_numpy().T)
fixelsh5.attrs['column_names'] = list(fixel_table.columns)
@@ -135,28 +115,28 @@ def mif_to_h5(
h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name])
f.close()
- return int(not output_hdf5.exists())
+ return int(not output.exists())
else:
- output_tiledb.mkdir(parents=True, exist_ok=True)
+ output.mkdir(parents=True, exist_ok=True)
for scalar_name in scalars.keys():
num_subjects = len(scalars[scalar_name])
num_items = scalars[scalar_name][0].shape[0] if num_subjects > 0 else 0
dataset_path = f'scalars/{scalar_name}/values'
tiledb_storage.create_empty_scalar_matrix_array(
- output_tiledb,
+ output,
dataset_path,
num_subjects,
num_items,
storage_dtype=storage_dtype,
- compression=tdb_compression,
- compression_level=tdb_compression_level,
- shuffle=tdb_shuffle,
- tile_voxels=tdb_tile_voxels,
- target_tile_mb=tdb_target_tile_mb,
+ compression=compression,
+ compression_level=compression_level,
+ shuffle=shuffle,
+ tile_voxels=chunk_voxels,
+ target_tile_mb=target_chunk_mb,
sources_list=sources_lists[scalar_name],
)
- uri = output_tiledb / dataset_path
+ uri = output / dataset_path
tiledb_storage.write_rows_in_column_stripes(uri, scalars[scalar_name])
return 0
@@ -167,19 +147,13 @@ def mif_to_h5_main(
directions_file,
cohort_file,
backend='hdf5',
- output_hdf5='fixelarray.h5',
- output_tiledb='arraydb.tdb',
+ output='fixelarray.h5',
storage_dtype='float32',
compression='gzip',
compression_level=4,
shuffle=True,
chunk_voxels=0,
target_chunk_mb=2.0,
- tdb_compression='zstd',
- tdb_compression_level=5,
- tdb_shuffle=True,
- tdb_tile_voxels=0,
- tdb_target_tile_mb=2.0,
log_level='INFO',
):
"""Entry point for the ``modelarrayio mif-to-h5`` command."""
@@ -192,19 +166,13 @@ def mif_to_h5_main(
directions_file=directions_file,
cohort_file=cohort_file,
backend=backend,
- output_hdf5=Path(output_hdf5),
- output_tiledb=Path(output_tiledb),
+ output=Path(output),
storage_dtype=storage_dtype,
compression=compression,
compression_level=compression_level,
shuffle=shuffle,
chunk_voxels=chunk_voxels,
target_chunk_mb=target_chunk_mb,
- tdb_compression=tdb_compression,
- tdb_compression_level=tdb_compression_level,
- tdb_shuffle=tdb_shuffle,
- tdb_tile_voxels=tdb_tile_voxels,
- tdb_target_tile_mb=tdb_target_tile_mb,
)
@@ -230,9 +198,7 @@ def _parse_mif_to_h5():
type=IsFile,
)
add_cohort_arg(parser)
- add_output_hdf5_arg(parser, default_name='fixelarray.h5')
- add_output_tiledb_arg(parser, default_name='arraydb.tdb')
+ add_output_arg(parser, default_name='fixelarray.h5')
add_backend_arg(parser)
add_storage_args(parser)
- add_tiledb_storage_args(parser)
return parser
diff --git a/src/modelarrayio/cli/nifti_to_h5.py b/src/modelarrayio/cli/nifti_to_h5.py
index c290290..d3a62f5 100644
--- a/src/modelarrayio/cli/nifti_to_h5.py
+++ b/src/modelarrayio/cli/nifti_to_h5.py
@@ -14,11 +14,9 @@
_is_file,
add_backend_arg,
add_cohort_arg,
- add_output_hdf5_arg,
- add_output_tiledb_arg,
+ add_output_arg,
add_s3_workers_arg,
add_storage_args,
- add_tiledb_storage_args,
)
from modelarrayio.storage import h5_storage, tiledb_storage
from modelarrayio.utils.voxels import _load_cohort_voxels
@@ -30,58 +28,50 @@ def nifti_to_h5(
group_mask_file,
cohort_file,
backend='hdf5',
- output_hdf5='voxeldb.h5',
- output_tiledb='arraydb.tdb',
+ output='voxeldb.h5',
storage_dtype='float32',
compression='gzip',
compression_level=4,
shuffle=True,
chunk_voxels=0,
target_chunk_mb=2.0,
- tdb_compression='zstd',
- tdb_compression_level=5,
- tdb_shuffle=True,
- tdb_tile_voxels=0,
- tdb_target_tile_mb=2.0,
s3_workers=1,
):
- """Load all volume data and write to an HDF5 file with configurable storage.
+ """Load all volume data and write to an HDF5 or TileDB file.
Parameters
----------
- group_mask_file: str
+ group_mask_file : :obj:`str`
Path to a NIfTI-1 binary group mask file.
- cohort_file: str
+ cohort_file : :obj:`str`
Path to a CSV with demographic info and paths to data.
- output_hdf5: str
- Path to a new .h5 file to be written.
- storage_dtype: str
- Floating type to store values. Options: 'float32' (default), 'float64'.
- compression: str
- HDF5 compression filter. Options: 'gzip' (default), 'lzf', 'none'.
- compression_level: int
- Gzip compression level (0-9). Only used when compression == 'gzip'. Default 4.
- shuffle: bool
- Enable HDF5 shuffle filter to improve compression.
- Default True (effective when compression != 'none').
- chunk_voxels: int
- Chunk size along the voxel axis. If 0, auto-compute using target_chunk_mb. Default 0.
- target_chunk_mb: float
- Target chunk size in MiB when auto-computing chunk_voxels. Default 2.0.
+ backend : :obj:`str`
+ Storage backend (``'hdf5'`` or ``'tiledb'``).
+ output : :obj:`str`
+ Output path. For the hdf5 backend, path to an .h5 file;
+ for the tiledb backend, path to a .tdb directory.
+ storage_dtype : :obj:`str`
+ Floating type to store values. Options: ``'float32'`` (default), ``'float64'``.
+ compression : :obj:`str`
+ Compression filter. ``gzip`` works for both backends;
+ ``lzf`` is HDF5-only; ``zstd`` is TileDB-only.
+ compression_level : :obj:`int`
+ Compression level (codec-dependent). Default 4.
+ shuffle : :obj:`bool`
+ Enable shuffle filter. Default True.
+ chunk_voxels : :obj:`int`
+ Chunk/tile size along the voxel axis. If 0, auto-compute. Default 0.
+ target_chunk_mb : :obj:`float`
+ Target chunk/tile size in MiB when auto-computing. Default 2.0.
+ s3_workers : :obj:`int`
+ Number of parallel workers for S3 downloads. Default 1.
"""
- # gather cohort data
cohort_df = pd.read_csv(cohort_file)
- # Load the group mask image to define the rows of the matrix
group_mask_img = nb.load(group_mask_file)
- # get_fdata(): get matrix data in float format
group_mask_matrix = group_mask_img.get_fdata() > 0
- # np.nonzero() returns the coords of nonzero elements;
- # then np.column_stack() stack them together as an (#voxels, 3) array
voxel_coords = np.column_stack(np.nonzero(group_mask_img.get_fdata()))
- # voxel_table: records the coordinations of the nonzero voxels;
- # coord starts from 0 (because using python)
voxel_table = pd.DataFrame(
{
'voxel_id': np.arange(voxel_coords.shape[0]),
@@ -91,16 +81,14 @@ def nifti_to_h5(
}
)
- # upload each cohort's data
print('Extracting NIfTI data...')
scalars, sources_lists = _load_cohort_voxels(cohort_df, group_mask_matrix, s3_workers)
- # Write the output:
if backend == 'hdf5':
- output_dir = os.path.dirname(output_hdf5)
- if not os.path.exists(output_dir):
+ output_dir = os.path.dirname(output)
+ if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir, exist_ok=True)
- f = h5py.File(output_hdf5, 'w')
+ f = h5py.File(output, 'w')
voxelsh5 = f.create_dataset(name='voxels', data=voxel_table.to_numpy().T)
voxelsh5.attrs['column_names'] = list(voxel_table.columns)
@@ -124,35 +112,28 @@ def nifti_to_h5(
h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name])
f.close()
- return int(not os.path.exists(output_hdf5))
+ return int(not os.path.exists(output))
else:
- # TileDB backend
- os.makedirs(output_tiledb, exist_ok=True)
-
- # Store voxel coordinates as a small TileDB array (optional):
- # we store as metadata on base group
- # Here we serialize as a dense 2D array for parity with HDF5 tables if desired,
- # but keep it simple: metadata JSON
- # Create arrays for each scalar
+ os.makedirs(output, exist_ok=True)
+
for scalar_name in scalars.keys():
num_subjects = len(scalars[scalar_name])
num_voxels = scalars[scalar_name][0].shape[0] if num_subjects > 0 else 0
dataset_path = f'scalars/{scalar_name}/values'
tiledb_storage.create_empty_scalar_matrix_array(
- output_tiledb,
+ output,
dataset_path,
num_subjects,
num_voxels,
storage_dtype=storage_dtype,
- compression=tdb_compression,
- compression_level=tdb_compression_level,
- shuffle=tdb_shuffle,
- tile_voxels=tdb_tile_voxels,
- target_tile_mb=tdb_target_tile_mb,
+ compression=compression,
+ compression_level=compression_level,
+ shuffle=shuffle,
+ tile_voxels=chunk_voxels,
+ target_tile_mb=target_chunk_mb,
sources_list=sources_lists[scalar_name],
)
- # Stripe-write
- uri = os.path.join(output_tiledb, dataset_path)
+ uri = os.path.join(output, dataset_path)
tiledb_storage.write_rows_in_column_stripes(uri, scalars[scalar_name])
return 0
@@ -161,19 +142,13 @@ def nifti_to_h5_main(
group_mask_file,
cohort_file,
backend='hdf5',
- output_hdf5='voxeldb.h5',
- output_tiledb='arraydb.tdb',
+ output='voxeldb.h5',
storage_dtype='float32',
compression='gzip',
compression_level=4,
shuffle=True,
chunk_voxels=0,
target_chunk_mb=2.0,
- tdb_compression='zstd',
- tdb_compression_level=5,
- tdb_shuffle=True,
- tdb_tile_voxels=0,
- tdb_target_tile_mb=2.0,
s3_workers=1,
log_level='INFO',
):
@@ -186,19 +161,13 @@ def nifti_to_h5_main(
group_mask_file=group_mask_file,
cohort_file=cohort_file,
backend=backend,
- output_hdf5=output_hdf5,
- output_tiledb=output_tiledb,
+ output=output,
storage_dtype=storage_dtype,
compression=compression,
compression_level=compression_level,
shuffle=shuffle,
chunk_voxels=chunk_voxels,
target_chunk_mb=target_chunk_mb,
- tdb_compression=tdb_compression,
- tdb_compression_level=tdb_compression_level,
- tdb_shuffle=tdb_shuffle,
- tdb_tile_voxels=tdb_tile_voxels,
- tdb_target_tile_mb=tdb_target_tile_mb,
s3_workers=s3_workers,
)
@@ -217,10 +186,8 @@ def _parse_nifti_to_h5():
type=IsFile,
)
add_cohort_arg(parser)
- add_output_hdf5_arg(parser, default_name='fixelarray.h5')
- add_output_tiledb_arg(parser, default_name='arraydb.tdb')
+ add_output_arg(parser, default_name='voxeldb.h5')
add_backend_arg(parser)
add_storage_args(parser)
- add_tiledb_storage_args(parser)
add_s3_workers_arg(parser)
return parser
diff --git a/src/modelarrayio/cli/parser_utils.py b/src/modelarrayio/cli/parser_utils.py
index 2ae0944..46e0bea 100644
--- a/src/modelarrayio/cli/parser_utils.py
+++ b/src/modelarrayio/cli/parser_utils.py
@@ -2,11 +2,13 @@
from pathlib import Path
-def add_output_hdf5_arg(parser, default_name='fixelarray.h5'):
+def add_output_arg(parser, default_name='output.h5'):
parser.add_argument(
- '--output-hdf5',
- '--output_hdf5',
- help='Name of HDF5 (.h5) file where outputs will be saved.',
+ '--output',
+ help=(
+ 'Output path. For the hdf5 backend, path to an .h5 file; '
+ 'for the tiledb backend, path to a .tdb directory.'
+ ),
default=default_name,
)
return parser
@@ -33,22 +35,27 @@ def add_storage_args(parser):
)
parser.add_argument(
'--compression',
- help='HDF5 compression filter: gzip (default), lzf, none',
- choices=['gzip', 'lzf', 'none'],
+ help=(
+ 'Compression filter (default gzip). '
+ 'gzip works for both backends; '
+ 'lzf is HDF5-only; '
+ 'zstd is TileDB-only.'
+ ),
+ choices=['gzip', 'zstd', 'lzf', 'none'],
default='gzip',
)
parser.add_argument(
'--compression-level',
'--compression_level',
type=int,
- help='Gzip compression level 0-9 (only if --compression=gzip). Default 4',
+ help='Compression level (codec-dependent). Default 4.',
default=4,
)
parser.add_argument(
'--no-shuffle',
dest='shuffle',
action='store_false',
- help='Disable HDF5 shuffle filter (enabled by default if compression is used).',
+ help='Disable shuffle filter (enabled by default when compression is used).',
default=True,
)
@@ -58,8 +65,8 @@ def add_storage_args(parser):
'--chunk_voxels',
type=int,
help=(
- 'Chunk size along voxel/greyordinate/fixel axis. If 0, auto-compute based on '
- '--target-chunk-mb and number of subjects'
+ 'Chunk/tile size along voxel/greyordinate/fixel axis. '
+ 'If 0, auto-compute based on --target-chunk-mb and number of subjects.'
),
default=0,
)
@@ -67,7 +74,7 @@ def add_storage_args(parser):
'--target-chunk-mb',
'--target_chunk_mb',
type=float,
- help='Target chunk size in MiB when auto-computing item chunk length. Default 2.0',
+ help='Target chunk/tile size in MiB when auto-computing the spatial axis length. Default 2.0.',
default=2.0,
)
@@ -92,60 +99,6 @@ def add_backend_arg(parser):
return parser
-def add_output_tiledb_arg(parser, default_name='arraydb.tdb'):
- parser.add_argument(
- '--output-tiledb',
- '--output_tiledb',
- help='Directory where TileDB arrays will be created.',
- default=default_name,
- )
- return parser
-
-
-def add_tiledb_storage_args(parser):
- parser.add_argument(
- '--tdb-compression',
- '--tdb_compression',
- help='TileDB compression: zstd (default), gzip, none',
- choices=['zstd', 'gzip', 'none'],
- default='zstd',
- )
- parser.add_argument(
- '--tdb-compression-level',
- '--tdb_compression_level',
- type=int,
- help='Compression level for TileDB (codec-dependent).',
- default=5,
- )
- parser.add_argument(
- '--tdb-no-shuffle',
- dest='tdb_shuffle',
- action='store_false',
- help='Disable TileDB shuffle filter (enabled by default).',
- default=True,
- )
-
- tile_allocation_group = parser.add_mutually_exclusive_group()
- tile_allocation_group.add_argument(
- '--tdb-tile-voxels',
- '--tdb_tile_voxels',
- type=int,
- help=(
- 'Tile length along item axis. If 0, auto-compute based on --tdb-target-tile-mb and '
- 'number of subjects'
- ),
- default=0,
- )
- tile_allocation_group.add_argument(
- '--tdb-target-tile-mb',
- '--tdb_target_tile_mb',
- type=float,
- help='Target tile size in MiB when auto-computing item tile length. Default 2.0',
- default=2.0,
- )
- return parser
-
-
def add_s3_workers_arg(parser):
parser.add_argument(
'--s3-workers',
diff --git a/test/test_cifti_cli.py b/test/test_cifti_cli.py
index 27a6039..2fb3fba 100644
--- a/test/test_cifti_cli.py
+++ b/test/test_cifti_cli.py
@@ -58,7 +58,7 @@ def test_concifti_cli_creates_expected_hdf5(tmp_path, monkeypatch):
'cifti-to-h5',
'--cohort-file',
str(cohort_csv),
- '--output-hdf5',
+ '--output',
str(out_h5),
'--backend',
'hdf5',
diff --git a/test/test_parser_utils.py b/test/test_parser_utils.py
index 1657fcf..c08a7c6 100644
--- a/test/test_parser_utils.py
+++ b/test/test_parser_utils.py
@@ -9,7 +9,7 @@
def _parser_with_cohort() -> argparse.ArgumentParser:
p = argparse.ArgumentParser()
- parser_utils.add_output_hdf5_arg(p)
+ parser_utils.add_output_arg(p)
parser_utils.add_cohort_arg(p)
parser_utils.add_storage_args(p)
parser_utils.add_backend_arg(p)
@@ -65,19 +65,19 @@ def test_storage_aliases_and_no_shuffle(tmp_path_factory) -> None:
def test_output_hdf5_default_name_override() -> None:
p = argparse.ArgumentParser()
- parser_utils.add_output_hdf5_arg(p, default_name='custom.h5')
+ parser_utils.add_output_arg(p, default_name='custom.h5')
args = p.parse_args([])
- assert args.output_hdf5 == 'custom.h5'
+ assert args.output == 'custom.h5'
def test_tiledb_args_group() -> None:
p = argparse.ArgumentParser()
- parser_utils.add_output_tiledb_arg(p, default_name='arrays.tdb')
- parser_utils.add_tiledb_storage_args(p)
+ parser_utils.add_output_arg(p, default_name='arrays.tdb')
+ parser_utils.add_storage_args(p)
args = p.parse_args([])
- assert args.output_tiledb == 'arrays.tdb'
- assert args.tdb_compression == 'zstd'
- assert args.tdb_tile_voxels == 0
+ assert args.output == 'arrays.tdb'
+ assert args.compression == 'gzip'
+ assert args.chunk_voxels == 0
def test_scalar_columns_optional(tmp_path_factory) -> None:
diff --git a/test/test_voxels_cli.py b/test/test_voxels_cli.py
index 3b883d7..4dcf958 100644
--- a/test/test_voxels_cli.py
+++ b/test/test_voxels_cli.py
@@ -79,7 +79,7 @@ def test_convoxel_cli_creates_expected_hdf5(tmp_path, monkeypatch):
str(group_mask_file),
'--cohort-file',
str(cohort_csv),
- '--output-hdf5',
+ '--output',
str(out_h5),
'--backend',
'hdf5',
diff --git a/test/test_voxels_s3.py b/test/test_voxels_s3.py
index 5660b13..16b7520 100644
--- a/test/test_voxels_s3.py
+++ b/test/test_voxels_s3.py
@@ -87,7 +87,7 @@ def test_convoxel_s3_parallel(tmp_path, group_mask_path, monkeypatch):
'group_mask.nii.gz',
'--cohort-file',
str(cohort_csv),
- '--output-hdf5',
+ '--output',
str(out_h5),
'--backend',
'hdf5',
@@ -159,9 +159,9 @@ def test_convoxel_s3_serial_matches_parallel(tmp_path, group_mask_path, monkeypa
monkeypatch.chdir(tmp_path)
monkeypatch.setenv('MODELARRAYIO_S3_ANON', '1')
for workers, name in [('1', 'serial.h5'), ('4', 'parallel.h5')]:
- assert (
- modelarrayio_main(base_argv + ['--output-hdf5', name, '--s3-workers', workers]) == 0
- ), f'modelarrayio nifti-to-h5 failed (workers={workers})'
+ assert modelarrayio_main(base_argv + ['--output', name, '--s3-workers', workers]) == 0, (
+ f'modelarrayio nifti-to-h5 failed (workers={workers})'
+ )
with (
h5py.File(tmp_path / 'serial.h5', 'r') as s,