From 54c1dd433e598e79d48f37051a03341388bdef5d Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 26 Mar 2026 11:19:18 -0400 Subject: [PATCH 1/3] Consolidate TileDB- and HDF5-related parameters. --- docs/examples/plot_fixel_workflow.py | 2 +- docs/examples/plot_voxel_workflow.py | 2 +- src/modelarrayio/cli/cifti_to_h5.py | 114 ++++++++++---------------- src/modelarrayio/cli/mif_to_h5.py | 90 +++++++-------------- src/modelarrayio/cli/nifti_to_h5.py | 115 ++++++++++----------------- src/modelarrayio/cli/parser_utils.py | 83 +++++-------------- test/test_cifti_cli.py | 2 +- test/test_voxels_cli.py | 2 +- 8 files changed, 133 insertions(+), 277 deletions(-) diff --git a/docs/examples/plot_fixel_workflow.py b/docs/examples/plot_fixel_workflow.py index 9f72a65..a350201 100644 --- a/docs/examples/plot_fixel_workflow.py +++ b/docs/examples/plot_fixel_workflow.py @@ -102,7 +102,7 @@ # --index-file /home/username/myProject/data/FD/index.mif \ # --directions-file /home/username/myProject/data/FD/directions.mif \ # --cohort-file /home/username/myProject/data/cohort_FD.csv \ -# --output-hdf5 /home/username/myProject/data/FD.h5 +# --output /home/username/myProject/data/FD.h5 # # This produces ``FD.h5`` in ``/home/username/myProject/data``. You can then use # `ModelArray `_ to run statistical analyses on it. diff --git a/docs/examples/plot_voxel_workflow.py b/docs/examples/plot_voxel_workflow.py index c0a810e..f4810c0 100644 --- a/docs/examples/plot_voxel_workflow.py +++ b/docs/examples/plot_voxel_workflow.py @@ -107,7 +107,7 @@ # modelarrayio nifti-to-h5 \ # --group-mask-file /home/username/myProject/data/group_mask.nii.gz \ # --cohort-file /home/username/myProject/data/cohort_FA.csv \ -# --output-hdf5 /home/username/myProject/data/FA.h5 +# --output /home/username/myProject/data/FA.h5 # # This produces ``FA.h5`` in ``/home/username/myProject/data``. You can then use # `ModelArray `_ to run statistical analyses on it. diff --git a/src/modelarrayio/cli/cifti_to_h5.py b/src/modelarrayio/cli/cifti_to_h5.py index 3dabd5c..50af90d 100644 --- a/src/modelarrayio/cli/cifti_to_h5.py +++ b/src/modelarrayio/cli/cifti_to_h5.py @@ -12,12 +12,10 @@ from modelarrayio.cli.parser_utils import ( add_backend_arg, add_cohort_arg, - add_output_hdf5_arg, - add_output_tiledb_arg, + add_output_arg, add_s3_workers_arg, add_scalar_columns_arg, add_storage_args, - add_tiledb_storage_args, ) from modelarrayio.storage import h5_storage, tiledb_storage from modelarrayio.utils.cifti import ( @@ -34,68 +32,53 @@ def cifti_to_h5( cohort_file, backend='hdf5', - output_hdf5='fixeldb.h5', - output_tiledb='arraydb.tdb', + output='fixelarray.h5', storage_dtype='float32', compression='gzip', compression_level=4, shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, - tdb_compression='zstd', - tdb_compression_level=5, - tdb_shuffle=True, - tdb_tile_voxels=0, - tdb_target_tile_mb=2.0, - tdb_workers=None, + workers=None, scalar_columns=None, s3_workers=1, ): - """Load all CIFTI data and write to an HDF5 file with configurable storage. + """Load all CIFTI data and write to an HDF5 or TileDB file. Parameters ---------- cohort_file : :obj:`str` Path to a csv with demographic info and paths to data backend : :obj:`str` - Backend to use for storage - output_hdf5 : :obj:`str` - Path to a new .h5 file to be written - output_tiledb : :obj:`str` - Path to a new .tdb file to be written + Backend to use for storage (``'hdf5'`` or ``'tiledb'``) + output : :obj:`str` + Output path. For the hdf5 backend, path to an .h5 file; + for the tiledb backend, path to a .tdb directory. storage_dtype : :obj:`str` Floating type to store values compression : :obj:`str` - HDF5 compression filter + Compression filter. ``gzip`` works for both backends; + ``lzf`` is HDF5-only; ``zstd`` is TileDB-only. compression_level : :obj:`int` - Gzip compression level (0-9) + Compression level (codec-dependent) shuffle : :obj:`bool` - Enable HDF5 shuffle filter + Enable shuffle filter chunk_voxels : :obj:`int` - Chunk size along the voxel axis + Chunk/tile size along the greyordinate axis (0 = auto) target_chunk_mb : :obj:`float` - Target chunk size in MiB when auto-computing chunk_voxels - tdb_compression : :obj:`str` - TileDB compression filter - tdb_compression_level : :obj:`int` - TileDB compression level - tdb_shuffle : :obj:`bool` - Enable TileDB shuffle filter - tdb_tile_voxels : :obj:`int` - Tile size along the voxel axis - tdb_target_tile_mb : :obj:`float` - Target tile size in MiB when auto-computing tdb_tile_voxels - tdb_workers : :obj:`int` - Number of workers to use for parallel loading + Target chunk/tile size in MiB when auto-computing the spatial axis length + workers : :obj:`int` + Maximum number of parallel TileDB write workers (``None`` = auto). + Has no effect when ``backend='hdf5'``. scalar_columns : :obj:`list` List of scalar columns to use s3_workers : :obj:`int` - Number of workers to use for parallel loading + Number of workers for parallel S3 downloads Returns ------- status : :obj:`int` - Status of the operation. 0 if successful, 1 if failed. + 0 if successful, 1 if failed. """ cohort_df = pd.read_csv(cohort_file) cohort_long = _cohort_to_long_dataframe(cohort_df, scalar_columns=scalar_columns) @@ -108,7 +91,7 @@ def cifti_to_h5( if backend == 'hdf5': scalars, last_brain_names = _load_cohort_cifti(cohort_long, s3_workers) - f = h5py.File(output_hdf5, 'w') + f = h5py.File(output, 'w') greyordinate_table, structure_names = brain_names_to_dataframe(last_brain_names) greyordinatesh5 = f.create_dataset( @@ -136,9 +119,9 @@ def cifti_to_h5( h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name]) f.close() - return int(not os.path.exists(output_hdf5)) + return int(not os.path.exists(output)) else: - os.makedirs(output_tiledb, exist_ok=True) + os.makedirs(output, exist_ok=True) if not scalar_sources: return 0 @@ -161,26 +144,26 @@ def _process_scalar_job(scalar_name, source_files): return scalar_name num_items = rows[0].shape[0] tiledb_storage.create_empty_scalar_matrix_array( - output_tiledb, + output, dataset_path, num_subjects, num_items, storage_dtype=storage_dtype, - compression=tdb_compression, - compression_level=tdb_compression_level, - shuffle=tdb_shuffle, - tile_voxels=tdb_tile_voxels, - target_tile_mb=tdb_target_tile_mb, + compression=compression, + compression_level=compression_level, + shuffle=shuffle, + tile_voxels=chunk_voxels, + target_tile_mb=target_chunk_mb, sources_list=source_files, ) # write column names array for ModelArray compatibility - tiledb_storage.write_column_names(output_tiledb, scalar_name, source_files) - uri = os.path.join(output_tiledb, dataset_path) + tiledb_storage.write_column_names(output, scalar_name, source_files) + uri = os.path.join(output, dataset_path) tiledb_storage.write_rows_in_column_stripes(uri, rows) return scalar_name scalar_names = list(scalar_sources.keys()) - worker_count = tdb_workers if isinstance(tdb_workers, int) and tdb_workers > 0 else None + worker_count = workers if isinstance(workers, int) and workers > 0 else None if worker_count is None: cpu_count = os.cpu_count() or 1 worker_count = min(len(scalar_names), max(1, cpu_count)) @@ -207,20 +190,14 @@ def _process_scalar_job(scalar_name, source_files): def cifti_to_h5_main( cohort_file, backend='hdf5', - output_hdf5='fixelarray.h5', - output_tiledb='arraydb.tdb', + output='fixelarray.h5', storage_dtype='float32', compression='gzip', compression_level=4, shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, - tdb_compression='zstd', - tdb_compression_level=5, - tdb_shuffle=True, - tdb_tile_voxels=0, - tdb_target_tile_mb=2.0, - tdb_workers=None, + workers=None, scalar_columns=None, s3_workers=1, log_level='INFO', @@ -233,20 +210,14 @@ def cifti_to_h5_main( return cifti_to_h5( cohort_file=cohort_file, backend=backend, - output_hdf5=output_hdf5, - output_tiledb=output_tiledb, + output=output, storage_dtype=storage_dtype, compression=compression, compression_level=compression_level, shuffle=shuffle, chunk_voxels=chunk_voxels, target_chunk_mb=target_chunk_mb, - tdb_compression=tdb_compression, - tdb_compression_level=tdb_compression_level, - tdb_shuffle=tdb_shuffle, - tdb_tile_voxels=tdb_tile_voxels, - tdb_target_tile_mb=tdb_target_tile_mb, - tdb_workers=tdb_workers, + workers=workers, scalar_columns=scalar_columns, s3_workers=s3_workers, ) @@ -254,23 +225,22 @@ def cifti_to_h5_main( def _parse_cifti_to_h5(): parser = argparse.ArgumentParser( - description='Create a hdf5 file of CIDTI2 dscalar data', + description='Create a hdf5 file of CIFTI2 dscalar data', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) add_cohort_arg(parser) add_scalar_columns_arg(parser) - add_output_hdf5_arg(parser, default_name='fixelarray.h5') - add_output_tiledb_arg(parser, default_name='arraydb.tdb') + add_output_arg(parser, default_name='fixelarray.h5') add_backend_arg(parser) add_storage_args(parser) - add_tiledb_storage_args(parser) parser.add_argument( - '--tdb-workers', - '--tdb_workers', + '--workers', type=int, help=( - 'Maximum number of TileDB write workers. Default 0 (auto, uses CPU count). ' - 'Set to 1 to disable parallel writes.' + 'Maximum number of parallel TileDB write workers. ' + 'Default 0 (auto, uses CPU count). ' + 'Set to 1 to disable parallel writes. ' + 'Has no effect when --backend=hdf5.' ), default=0, ) diff --git a/src/modelarrayio/cli/mif_to_h5.py b/src/modelarrayio/cli/mif_to_h5.py index d2f95e8..9a55bb6 100644 --- a/src/modelarrayio/cli/mif_to_h5.py +++ b/src/modelarrayio/cli/mif_to_h5.py @@ -14,10 +14,8 @@ _is_file, add_backend_arg, add_cohort_arg, - add_output_hdf5_arg, - add_output_tiledb_arg, + add_output_arg, add_storage_args, - add_tiledb_storage_args, ) from modelarrayio.storage import h5_storage, tiledb_storage from modelarrayio.utils.fixels import gather_fixels, mif_to_nifti2 @@ -30,21 +28,15 @@ def mif_to_h5( directions_file, cohort_file, backend='hdf5', - output_hdf5=Path('fixeldb.h5'), - output_tiledb=Path('arraydb.tdb'), + output=Path('fixelarray.h5'), storage_dtype='float32', compression='gzip', compression_level=4, shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, - tdb_compression='zstd', - tdb_compression_level=5, - tdb_shuffle=True, - tdb_tile_voxels=0, - tdb_target_tile_mb=2.0, ): - """Load all fixeldb data and write to an HDF5 file with configurable storage. + """Load all fixeldb data and write to an HDF5 or TileDB file. Parameters ---------- @@ -55,38 +47,28 @@ def mif_to_h5( cohort_file : :obj:`pathlib.Path` Path to a csv with demographic info and paths to data backend : :obj:`str` - Backend to use for storage - output_hdf5 : :obj:`pathlib.Path` - Path to a new .h5 file to be written - output_tiledb : :obj:`pathlib.Path` - Path to a new .tdb file to be written + Backend to use for storage (``'hdf5'`` or ``'tiledb'``) + output : :obj:`pathlib.Path` + Output path. For the hdf5 backend, path to an .h5 file; + for the tiledb backend, path to a .tdb directory. storage_dtype : :obj:`str` Floating type to store values compression : :obj:`str` - HDF5 compression filter + Compression filter. ``gzip`` works for both backends; + ``lzf`` is HDF5-only; ``zstd`` is TileDB-only. compression_level : :obj:`int` - Gzip compression level (0-9) + Compression level (codec-dependent) shuffle : :obj:`bool` - Enable HDF5 shuffle filter + Enable shuffle filter chunk_voxels : :obj:`int` - Chunk size along the voxel axis + Chunk/tile size along the fixel axis (0 = auto) target_chunk_mb : :obj:`float` - Target chunk size in MiB when auto-computing chunk_voxels - tdb_compression : :obj:`str` - TileDB compression filter - tdb_compression_level : :obj:`int` - TileDB compression level - tdb_shuffle : :obj:`bool` - Enable TileDB shuffle filter - tdb_tile_voxels : :obj:`int` - Tile size along the voxel axis - tdb_target_tile_mb : :obj:`float` - Target tile size in MiB when auto-computing tdb_tile_voxels + Target chunk/tile size in MiB when auto-computing the spatial axis length Returns ------- status : :obj:`int` - Status of the operation. 0 if successful, 1 if failed. + 0 if successful, 1 if failed. """ # gather fixel data fixel_table, voxel_table = gather_fixels(index_file, directions_file) @@ -98,17 +80,15 @@ def mif_to_h5( scalars = defaultdict(list) sources_lists = defaultdict(list) print('Extracting .mif data...') - # ix: index of row (start from 0); row: one row of data for _ix, row in tqdm(cohort_df.iterrows(), total=cohort_df.shape[0]): scalar_file = row['source_file'] _scalar_img, scalar_data = mif_to_nifti2(scalar_file) - scalars[row['scalar_name']].append(scalar_data) # append to specific scalar_name - # append source mif filename to specific scalar_name + scalars[row['scalar_name']].append(scalar_data) sources_lists[row['scalar_name']].append(row['source_file']) # Write the output if backend == 'hdf5': - f = h5py.File(output_hdf5, 'w') + f = h5py.File(output, 'w') fixelsh5 = f.create_dataset(name='fixels', data=fixel_table.to_numpy().T) fixelsh5.attrs['column_names'] = list(fixel_table.columns) @@ -135,28 +115,28 @@ def mif_to_h5( h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name]) f.close() - return int(not output_hdf5.exists()) + return int(not output.exists()) else: - output_tiledb.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=True) for scalar_name in scalars.keys(): num_subjects = len(scalars[scalar_name]) num_items = scalars[scalar_name][0].shape[0] if num_subjects > 0 else 0 dataset_path = f'scalars/{scalar_name}/values' tiledb_storage.create_empty_scalar_matrix_array( - output_tiledb, + output, dataset_path, num_subjects, num_items, storage_dtype=storage_dtype, - compression=tdb_compression, - compression_level=tdb_compression_level, - shuffle=tdb_shuffle, - tile_voxels=tdb_tile_voxels, - target_tile_mb=tdb_target_tile_mb, + compression=compression, + compression_level=compression_level, + shuffle=shuffle, + tile_voxels=chunk_voxels, + target_tile_mb=target_chunk_mb, sources_list=sources_lists[scalar_name], ) - uri = output_tiledb / dataset_path + uri = output / dataset_path tiledb_storage.write_rows_in_column_stripes(uri, scalars[scalar_name]) return 0 @@ -167,19 +147,13 @@ def mif_to_h5_main( directions_file, cohort_file, backend='hdf5', - output_hdf5='fixelarray.h5', - output_tiledb='arraydb.tdb', + output='fixelarray.h5', storage_dtype='float32', compression='gzip', compression_level=4, shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, - tdb_compression='zstd', - tdb_compression_level=5, - tdb_shuffle=True, - tdb_tile_voxels=0, - tdb_target_tile_mb=2.0, log_level='INFO', ): """Entry point for the ``modelarrayio mif-to-h5`` command.""" @@ -192,19 +166,13 @@ def mif_to_h5_main( directions_file=directions_file, cohort_file=cohort_file, backend=backend, - output_hdf5=Path(output_hdf5), - output_tiledb=Path(output_tiledb), + output=Path(output), storage_dtype=storage_dtype, compression=compression, compression_level=compression_level, shuffle=shuffle, chunk_voxels=chunk_voxels, target_chunk_mb=target_chunk_mb, - tdb_compression=tdb_compression, - tdb_compression_level=tdb_compression_level, - tdb_shuffle=tdb_shuffle, - tdb_tile_voxels=tdb_tile_voxels, - tdb_target_tile_mb=tdb_target_tile_mb, ) @@ -230,9 +198,7 @@ def _parse_mif_to_h5(): type=IsFile, ) add_cohort_arg(parser) - add_output_hdf5_arg(parser, default_name='fixelarray.h5') - add_output_tiledb_arg(parser, default_name='arraydb.tdb') + add_output_arg(parser, default_name='fixelarray.h5') add_backend_arg(parser) add_storage_args(parser) - add_tiledb_storage_args(parser) return parser diff --git a/src/modelarrayio/cli/nifti_to_h5.py b/src/modelarrayio/cli/nifti_to_h5.py index c290290..d3a62f5 100644 --- a/src/modelarrayio/cli/nifti_to_h5.py +++ b/src/modelarrayio/cli/nifti_to_h5.py @@ -14,11 +14,9 @@ _is_file, add_backend_arg, add_cohort_arg, - add_output_hdf5_arg, - add_output_tiledb_arg, + add_output_arg, add_s3_workers_arg, add_storage_args, - add_tiledb_storage_args, ) from modelarrayio.storage import h5_storage, tiledb_storage from modelarrayio.utils.voxels import _load_cohort_voxels @@ -30,58 +28,50 @@ def nifti_to_h5( group_mask_file, cohort_file, backend='hdf5', - output_hdf5='voxeldb.h5', - output_tiledb='arraydb.tdb', + output='voxeldb.h5', storage_dtype='float32', compression='gzip', compression_level=4, shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, - tdb_compression='zstd', - tdb_compression_level=5, - tdb_shuffle=True, - tdb_tile_voxels=0, - tdb_target_tile_mb=2.0, s3_workers=1, ): - """Load all volume data and write to an HDF5 file with configurable storage. + """Load all volume data and write to an HDF5 or TileDB file. Parameters ---------- - group_mask_file: str + group_mask_file : :obj:`str` Path to a NIfTI-1 binary group mask file. - cohort_file: str + cohort_file : :obj:`str` Path to a CSV with demographic info and paths to data. - output_hdf5: str - Path to a new .h5 file to be written. - storage_dtype: str - Floating type to store values. Options: 'float32' (default), 'float64'. - compression: str - HDF5 compression filter. Options: 'gzip' (default), 'lzf', 'none'. - compression_level: int - Gzip compression level (0-9). Only used when compression == 'gzip'. Default 4. - shuffle: bool - Enable HDF5 shuffle filter to improve compression. - Default True (effective when compression != 'none'). - chunk_voxels: int - Chunk size along the voxel axis. If 0, auto-compute using target_chunk_mb. Default 0. - target_chunk_mb: float - Target chunk size in MiB when auto-computing chunk_voxels. Default 2.0. + backend : :obj:`str` + Storage backend (``'hdf5'`` or ``'tiledb'``). + output : :obj:`str` + Output path. For the hdf5 backend, path to an .h5 file; + for the tiledb backend, path to a .tdb directory. + storage_dtype : :obj:`str` + Floating type to store values. Options: ``'float32'`` (default), ``'float64'``. + compression : :obj:`str` + Compression filter. ``gzip`` works for both backends; + ``lzf`` is HDF5-only; ``zstd`` is TileDB-only. + compression_level : :obj:`int` + Compression level (codec-dependent). Default 4. + shuffle : :obj:`bool` + Enable shuffle filter. Default True. + chunk_voxels : :obj:`int` + Chunk/tile size along the voxel axis. If 0, auto-compute. Default 0. + target_chunk_mb : :obj:`float` + Target chunk/tile size in MiB when auto-computing. Default 2.0. + s3_workers : :obj:`int` + Number of parallel workers for S3 downloads. Default 1. """ - # gather cohort data cohort_df = pd.read_csv(cohort_file) - # Load the group mask image to define the rows of the matrix group_mask_img = nb.load(group_mask_file) - # get_fdata(): get matrix data in float format group_mask_matrix = group_mask_img.get_fdata() > 0 - # np.nonzero() returns the coords of nonzero elements; - # then np.column_stack() stack them together as an (#voxels, 3) array voxel_coords = np.column_stack(np.nonzero(group_mask_img.get_fdata())) - # voxel_table: records the coordinations of the nonzero voxels; - # coord starts from 0 (because using python) voxel_table = pd.DataFrame( { 'voxel_id': np.arange(voxel_coords.shape[0]), @@ -91,16 +81,14 @@ def nifti_to_h5( } ) - # upload each cohort's data print('Extracting NIfTI data...') scalars, sources_lists = _load_cohort_voxels(cohort_df, group_mask_matrix, s3_workers) - # Write the output: if backend == 'hdf5': - output_dir = os.path.dirname(output_hdf5) - if not os.path.exists(output_dir): + output_dir = os.path.dirname(output) + if output_dir and not os.path.exists(output_dir): os.makedirs(output_dir, exist_ok=True) - f = h5py.File(output_hdf5, 'w') + f = h5py.File(output, 'w') voxelsh5 = f.create_dataset(name='voxels', data=voxel_table.to_numpy().T) voxelsh5.attrs['column_names'] = list(voxel_table.columns) @@ -124,35 +112,28 @@ def nifti_to_h5( h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name]) f.close() - return int(not os.path.exists(output_hdf5)) + return int(not os.path.exists(output)) else: - # TileDB backend - os.makedirs(output_tiledb, exist_ok=True) - - # Store voxel coordinates as a small TileDB array (optional): - # we store as metadata on base group - # Here we serialize as a dense 2D array for parity with HDF5 tables if desired, - # but keep it simple: metadata JSON - # Create arrays for each scalar + os.makedirs(output, exist_ok=True) + for scalar_name in scalars.keys(): num_subjects = len(scalars[scalar_name]) num_voxels = scalars[scalar_name][0].shape[0] if num_subjects > 0 else 0 dataset_path = f'scalars/{scalar_name}/values' tiledb_storage.create_empty_scalar_matrix_array( - output_tiledb, + output, dataset_path, num_subjects, num_voxels, storage_dtype=storage_dtype, - compression=tdb_compression, - compression_level=tdb_compression_level, - shuffle=tdb_shuffle, - tile_voxels=tdb_tile_voxels, - target_tile_mb=tdb_target_tile_mb, + compression=compression, + compression_level=compression_level, + shuffle=shuffle, + tile_voxels=chunk_voxels, + target_tile_mb=target_chunk_mb, sources_list=sources_lists[scalar_name], ) - # Stripe-write - uri = os.path.join(output_tiledb, dataset_path) + uri = os.path.join(output, dataset_path) tiledb_storage.write_rows_in_column_stripes(uri, scalars[scalar_name]) return 0 @@ -161,19 +142,13 @@ def nifti_to_h5_main( group_mask_file, cohort_file, backend='hdf5', - output_hdf5='voxeldb.h5', - output_tiledb='arraydb.tdb', + output='voxeldb.h5', storage_dtype='float32', compression='gzip', compression_level=4, shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, - tdb_compression='zstd', - tdb_compression_level=5, - tdb_shuffle=True, - tdb_tile_voxels=0, - tdb_target_tile_mb=2.0, s3_workers=1, log_level='INFO', ): @@ -186,19 +161,13 @@ def nifti_to_h5_main( group_mask_file=group_mask_file, cohort_file=cohort_file, backend=backend, - output_hdf5=output_hdf5, - output_tiledb=output_tiledb, + output=output, storage_dtype=storage_dtype, compression=compression, compression_level=compression_level, shuffle=shuffle, chunk_voxels=chunk_voxels, target_chunk_mb=target_chunk_mb, - tdb_compression=tdb_compression, - tdb_compression_level=tdb_compression_level, - tdb_shuffle=tdb_shuffle, - tdb_tile_voxels=tdb_tile_voxels, - tdb_target_tile_mb=tdb_target_tile_mb, s3_workers=s3_workers, ) @@ -217,10 +186,8 @@ def _parse_nifti_to_h5(): type=IsFile, ) add_cohort_arg(parser) - add_output_hdf5_arg(parser, default_name='fixelarray.h5') - add_output_tiledb_arg(parser, default_name='arraydb.tdb') + add_output_arg(parser, default_name='voxeldb.h5') add_backend_arg(parser) add_storage_args(parser) - add_tiledb_storage_args(parser) add_s3_workers_arg(parser) return parser diff --git a/src/modelarrayio/cli/parser_utils.py b/src/modelarrayio/cli/parser_utils.py index 2ae0944..46e0bea 100644 --- a/src/modelarrayio/cli/parser_utils.py +++ b/src/modelarrayio/cli/parser_utils.py @@ -2,11 +2,13 @@ from pathlib import Path -def add_output_hdf5_arg(parser, default_name='fixelarray.h5'): +def add_output_arg(parser, default_name='output.h5'): parser.add_argument( - '--output-hdf5', - '--output_hdf5', - help='Name of HDF5 (.h5) file where outputs will be saved.', + '--output', + help=( + 'Output path. For the hdf5 backend, path to an .h5 file; ' + 'for the tiledb backend, path to a .tdb directory.' + ), default=default_name, ) return parser @@ -33,22 +35,27 @@ def add_storage_args(parser): ) parser.add_argument( '--compression', - help='HDF5 compression filter: gzip (default), lzf, none', - choices=['gzip', 'lzf', 'none'], + help=( + 'Compression filter (default gzip). ' + 'gzip works for both backends; ' + 'lzf is HDF5-only; ' + 'zstd is TileDB-only.' + ), + choices=['gzip', 'zstd', 'lzf', 'none'], default='gzip', ) parser.add_argument( '--compression-level', '--compression_level', type=int, - help='Gzip compression level 0-9 (only if --compression=gzip). Default 4', + help='Compression level (codec-dependent). Default 4.', default=4, ) parser.add_argument( '--no-shuffle', dest='shuffle', action='store_false', - help='Disable HDF5 shuffle filter (enabled by default if compression is used).', + help='Disable shuffle filter (enabled by default when compression is used).', default=True, ) @@ -58,8 +65,8 @@ def add_storage_args(parser): '--chunk_voxels', type=int, help=( - 'Chunk size along voxel/greyordinate/fixel axis. If 0, auto-compute based on ' - '--target-chunk-mb and number of subjects' + 'Chunk/tile size along voxel/greyordinate/fixel axis. ' + 'If 0, auto-compute based on --target-chunk-mb and number of subjects.' ), default=0, ) @@ -67,7 +74,7 @@ def add_storage_args(parser): '--target-chunk-mb', '--target_chunk_mb', type=float, - help='Target chunk size in MiB when auto-computing item chunk length. Default 2.0', + help='Target chunk/tile size in MiB when auto-computing the spatial axis length. Default 2.0.', default=2.0, ) @@ -92,60 +99,6 @@ def add_backend_arg(parser): return parser -def add_output_tiledb_arg(parser, default_name='arraydb.tdb'): - parser.add_argument( - '--output-tiledb', - '--output_tiledb', - help='Directory where TileDB arrays will be created.', - default=default_name, - ) - return parser - - -def add_tiledb_storage_args(parser): - parser.add_argument( - '--tdb-compression', - '--tdb_compression', - help='TileDB compression: zstd (default), gzip, none', - choices=['zstd', 'gzip', 'none'], - default='zstd', - ) - parser.add_argument( - '--tdb-compression-level', - '--tdb_compression_level', - type=int, - help='Compression level for TileDB (codec-dependent).', - default=5, - ) - parser.add_argument( - '--tdb-no-shuffle', - dest='tdb_shuffle', - action='store_false', - help='Disable TileDB shuffle filter (enabled by default).', - default=True, - ) - - tile_allocation_group = parser.add_mutually_exclusive_group() - tile_allocation_group.add_argument( - '--tdb-tile-voxels', - '--tdb_tile_voxels', - type=int, - help=( - 'Tile length along item axis. If 0, auto-compute based on --tdb-target-tile-mb and ' - 'number of subjects' - ), - default=0, - ) - tile_allocation_group.add_argument( - '--tdb-target-tile-mb', - '--tdb_target_tile_mb', - type=float, - help='Target tile size in MiB when auto-computing item tile length. Default 2.0', - default=2.0, - ) - return parser - - def add_s3_workers_arg(parser): parser.add_argument( '--s3-workers', diff --git a/test/test_cifti_cli.py b/test/test_cifti_cli.py index 27a6039..2fb3fba 100644 --- a/test/test_cifti_cli.py +++ b/test/test_cifti_cli.py @@ -58,7 +58,7 @@ def test_concifti_cli_creates_expected_hdf5(tmp_path, monkeypatch): 'cifti-to-h5', '--cohort-file', str(cohort_csv), - '--output-hdf5', + '--output', str(out_h5), '--backend', 'hdf5', diff --git a/test/test_voxels_cli.py b/test/test_voxels_cli.py index 3b883d7..4dcf958 100644 --- a/test/test_voxels_cli.py +++ b/test/test_voxels_cli.py @@ -79,7 +79,7 @@ def test_convoxel_cli_creates_expected_hdf5(tmp_path, monkeypatch): str(group_mask_file), '--cohort-file', str(cohort_csv), - '--output-hdf5', + '--output', str(out_h5), '--backend', 'hdf5', From 361d2b3e4a1dd5135c9ced2a558c1dd8fef9cde8 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 26 Mar 2026 11:23:51 -0400 Subject: [PATCH 2/3] Fix tests. --- test/test_parser_utils.py | 16 ++++++++-------- test/test_voxels_s3.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/test_parser_utils.py b/test/test_parser_utils.py index 1657fcf..c08a7c6 100644 --- a/test/test_parser_utils.py +++ b/test/test_parser_utils.py @@ -9,7 +9,7 @@ def _parser_with_cohort() -> argparse.ArgumentParser: p = argparse.ArgumentParser() - parser_utils.add_output_hdf5_arg(p) + parser_utils.add_output_arg(p) parser_utils.add_cohort_arg(p) parser_utils.add_storage_args(p) parser_utils.add_backend_arg(p) @@ -65,19 +65,19 @@ def test_storage_aliases_and_no_shuffle(tmp_path_factory) -> None: def test_output_hdf5_default_name_override() -> None: p = argparse.ArgumentParser() - parser_utils.add_output_hdf5_arg(p, default_name='custom.h5') + parser_utils.add_output_arg(p, default_name='custom.h5') args = p.parse_args([]) - assert args.output_hdf5 == 'custom.h5' + assert args.output == 'custom.h5' def test_tiledb_args_group() -> None: p = argparse.ArgumentParser() - parser_utils.add_output_tiledb_arg(p, default_name='arrays.tdb') - parser_utils.add_tiledb_storage_args(p) + parser_utils.add_output_arg(p, default_name='arrays.tdb') + parser_utils.add_storage_args(p) args = p.parse_args([]) - assert args.output_tiledb == 'arrays.tdb' - assert args.tdb_compression == 'zstd' - assert args.tdb_tile_voxels == 0 + assert args.output == 'arrays.tdb' + assert args.compression == 'gzip' + assert args.chunk_voxels == 0 def test_scalar_columns_optional(tmp_path_factory) -> None: diff --git a/test/test_voxels_s3.py b/test/test_voxels_s3.py index 5660b13..27aae91 100644 --- a/test/test_voxels_s3.py +++ b/test/test_voxels_s3.py @@ -87,7 +87,7 @@ def test_convoxel_s3_parallel(tmp_path, group_mask_path, monkeypatch): 'group_mask.nii.gz', '--cohort-file', str(cohort_csv), - '--output-hdf5', + '--output', str(out_h5), '--backend', 'hdf5', @@ -160,7 +160,7 @@ def test_convoxel_s3_serial_matches_parallel(tmp_path, group_mask_path, monkeypa monkeypatch.setenv('MODELARRAYIO_S3_ANON', '1') for workers, name in [('1', 'serial.h5'), ('4', 'parallel.h5')]: assert ( - modelarrayio_main(base_argv + ['--output-hdf5', name, '--s3-workers', workers]) == 0 + modelarrayio_main(base_argv + ['--output', name, '--s3-workers', workers]) == 0 ), f'modelarrayio nifti-to-h5 failed (workers={workers})' with ( From a8ffcff129ce78863451c007664e4421fa2363db Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Thu, 26 Mar 2026 11:24:13 -0400 Subject: [PATCH 3/3] Update test_voxels_s3.py --- test/test_voxels_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_voxels_s3.py b/test/test_voxels_s3.py index 27aae91..16b7520 100644 --- a/test/test_voxels_s3.py +++ b/test/test_voxels_s3.py @@ -159,9 +159,9 @@ def test_convoxel_s3_serial_matches_parallel(tmp_path, group_mask_path, monkeypa monkeypatch.chdir(tmp_path) monkeypatch.setenv('MODELARRAYIO_S3_ANON', '1') for workers, name in [('1', 'serial.h5'), ('4', 'parallel.h5')]: - assert ( - modelarrayio_main(base_argv + ['--output', name, '--s3-workers', workers]) == 0 - ), f'modelarrayio nifti-to-h5 failed (workers={workers})' + assert modelarrayio_main(base_argv + ['--output', name, '--s3-workers', workers]) == 0, ( + f'modelarrayio nifti-to-h5 failed (workers={workers})' + ) with ( h5py.File(tmp_path / 'serial.h5', 'r') as s,