diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..407c5a5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,4 @@ +contact_links: + - name: Usage question + url: https://neurostars.org/tags/c/software-support/234/modelarray + about: Please ask questions about using ModelArrayIO on NeuroStars. diff --git a/README.rst b/README.rst index 26bfc12..e6f01a1 100644 --- a/README.rst +++ b/README.rst @@ -50,70 +50,19 @@ Once ModelArrayIO is installed, these commands are available in your terminal: * **Fixel-wise** data (MRtrix ``.mif``): - * ``.mif`` → ``.h5``: ``confixel`` (CLI name kept for compatibility with earlier ConFixel workflows) - * ``.h5`` → ``.mif``: ``fixelstats_write`` + * ``.mif`` → ``.h5``: ``modelarrayio mif-to-h5`` + * ``.h5`` → ``.mif``: ``modelarrayio h5-to-mif`` * **Voxel-wise** data (NIfTI): - * NIfTI → ``.h5``: ``convoxel`` - * ``.h5`` → NIfTI: ``volumestats_write`` + * NIfTI → ``.h5``: ``modelarrayio nifti-to-h5`` + * ``.h5`` → NIfTI: ``modelarrayio h5-to-nifti`` * **Greyordinate-wise** data (CIFTI-2): - * CIFTI-2 → ``.h5``: ``concifti`` - * ``.h5`` → CIFTI-2: ``ciftistats_write`` + * CIFTI-2 → ``.h5``: ``modelarrayio cifti-to-h5`` + * ``.h5`` → CIFTI-2: ``modelarrayio h5-to-cifti`` -Installation -============ - -MRtrix (required for fixel ``.mif`` only) ------------------------------------------ - -For fixel-wise ``.mif`` conversion, the ``confixel`` / ``fixelstats_write`` tools use MRtrix ``mrconvert``. Install MRtrix from `MRtrix’s webpage `_ if needed. Run ``mrview`` in the terminal to verify the installation. - -If your data are voxel-wise or CIFTI only, you can skip this step. - -Install ModelArrayIO --------------------- - -You may want a conda environment first—see `ModelArray: Installation `_. If MRtrix is installed in that environment, install ModelArrayIO in the same environment. - -Install from GitHub: - -.. code-block:: console - - git clone https://github.com/PennLINC/ModelArrayIO.git - cd ModelArrayIO - pip install . # build via pyproject.toml - -Editable install for development: - -.. code-block:: console - - # From the repository root - pip install -e . - -With ``hatch`` installed, you can build wheels/sdist locally: - -.. code-block:: console - - hatch build - pip install dist/*.whl - -How to use -========== - -We provide a `walkthrough for fixel-wise data `_ (``confixel`` / ``fixelstats_write``) and a `walkthrough for voxel-wise data `_ (``convoxel`` / ``volumestats_write``). - -Together with `ModelArray `_, see the `combined walkthrough `_ with example fixel-wise data (ModelArray + ModelArrayIO). - -CLI help: - -.. code-block:: console - - confixel --help - -Use the same pattern for ``convoxel``, ``concifti``, ``fixelstats_write``, ``volumestats_write``, and ``ciftistats_write``. Storage backends: HDF5 and TileDB ================================= diff --git a/docs/examples/plot_cifti_workflow.py b/docs/examples/plot_cifti_workflow.py new file mode 100644 index 0000000..1bdc221 --- /dev/null +++ b/docs/examples/plot_cifti_workflow.py @@ -0,0 +1,167 @@ +""" +CIFTI (Greyordinate-wise) Data Conversion +========================================= + +For imaging data in CIFTI format, use the ``modelarrayio cifti-to-h5`` command to convert +the CIFTI files to the HDF5 format (``.h5``) used by **ModelArray**, +and ``modelarrayio h5-to-cifti`` to export results back to CIFTI. +The CIFTI workflow is very similar to the MIF workflow +(:ref:`sphx_glr_auto_examples_plot_mif_workflow.py`). +""" + +# %% +# Prepare data +# ------------ +# +# To convert a list of CIFTI files to ``.h5`` format, you need: +# +# 1. **A cohort CSV** describing every CIFTI file to include (one CSV per scalar recommended). +# +# Cohort CSV columns (names are fixed, not user-defined): +# +# * ``scalar_name`` — which metric is being analysed (e.g., ``FA``) +# * ``source_file`` — path to the subject's CIFTI file + +# %% +# Example folder structure +# ------------------------ +# +# .. code-block:: text +# +# /home/username/myProject/data +# | +# ├── cohort_FA.csv +# │ +# ├── FA +# │ ├── sub-01_FA.dscalar.nii +# │ ├── sub-02_FA.dscalar.nii +# │ ├── sub-03_FA.dscalar.nii +# │ └── ... +# │ +# └── ... +# +# Corresponding ``cohort_FA.csv`` for scalar FA: +# +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - **scalar_name** *(required)* +# - **source_file** *(required)* +# - subject_id +# - age +# - sex +# * - FA +# - /home/username/myProject/data/FA/sub-01_FA.dscalar.nii +# - sub-01 +# - 10 +# - F +# * - FA +# - /home/username/myProject/data/FA/sub-02_FA.dscalar.nii +# - sub-02 +# - 20 +# - M +# * - FA +# - /home/username/myProject/data/FA/sub-03_FA.dscalar.nii +# - sub-03 +# - 15 +# - F +# * - ... +# - ... +# - ... +# - ... +# - ... +# +# Notes: +# +# * Column order does not matter. +# * Values are case-sensitive — folder names, file names, and scalar names must match exactly +# between the CSV and disk. + +# %% +# Convert CIFTI files to HDF5 +# --------------------------- +# +# Using the FA dataset from the example above: +# +# .. code-block:: console +# +# # activate your conda environment first +# conda activate +# +# modelarrayio cifti-to-h5 \ +# --cohort-file /home/username/myProject/data/cohort_FA.csv \ +# --output /home/username/myProject/data/FA.h5 +# +# This produces ``FA.h5`` in ``/home/username/myProject/data``. You can then use +# `ModelArray `_ to run statistical analyses on it. + +# %% +# Convert result .h5 back to CIFTI +# -------------------------------- +# +# After running **ModelArray** and obtaining statistical results inside ``FA.h5`` (suppose the +# analysis name is ``"mylm"``), use ``modelarrayio h5-to-cifti`` to export them as CIFTI files. +# +# You must also provide an example CIFTI file to use as a template for the output. +# +# .. code-block:: console +# +# modelarrayio h5-to-cifti \ +# --cohort-file /home/username/myProject/data/cohort_FA.csv \ +# --analysis-name mylm \ +# --input-hdf5 /home/username/myProject/data/FA.h5 \ +# --output-dir /home/username/myProject/data/FA_stats \ +# --example-cifti /home/username/myProject/data/FA/sub-01_FA.dscalar.nii +# +# All converted volume data are saved as ``float32``. Results in ``FA_stats`` can be viewed +# with any CIFTI image viewer. +# +# .. warning:: +# +# If ``--output-dir`` already exists, ``modelarrayio h5-to-cifti`` will not delete it — you will +# see ``WARNING: Output directory exists``. Existing files that are **not** part of the +# current output list are left unchanged. Existing files that **are** part of the current +# output list will be overwritten. To avoid confusion, consider manually deleting the output +# directory before re-running ``modelarrayio h5-to-cifti``. + +# %% +# Number-of-observations image +# ---------------------------- +# +# If you requested ``nobs`` during model fitting in ModelArray, after conversion you will find +# an image called ``*_model.nobs.dscalar.nii*``. With subject-specific masks, this image may be +# inhomogeneous across voxels. +# +# Voxels that did not have sufficient subjects (due to subject-specific masking) are stored as +# ``NaN`` in the HDF5 file. How different viewers display these greyordinates: +# +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - Viewer +# - Regular voxel +# - Voxel without sufficient subjects +# * - nibabel (Python) +# - e.g. ``nobs: 209.0``, ``p.value: 0.01`` +# - ``NaN`` +# * - MRtrix mrview +# - e.g. ``nobs: 209``, ``p.value: 0.01`` +# - ``?`` +# * - ITK-SNAP +# - e.g. ``nobs: 209``, ``p.value: 0.01`` +# - ``0`` (displayed, but excluded when thresholding) + +# %% +# Additional help +# --------------- +# +# Full argument documentation is available from the command line: +# +# .. code-block:: console +# +# modelarrayio cifti-to-h5 --help +# modelarrayio h5-to-cifti --help +# +# or in the :doc:`/usage` page of this documentation. diff --git a/docs/examples/plot_fixel_workflow.py b/docs/examples/plot_fixel_workflow.py index a350201..3ee7fcb 100644 --- a/docs/examples/plot_fixel_workflow.py +++ b/docs/examples/plot_fixel_workflow.py @@ -1,10 +1,12 @@ """ -Fixel-wise Data Conversion -========================== +MIF (Fixel-wise) Data Conversion +================================ -For fixel-wise data, use the **``confixel``** command from **ModelArrayIO** to convert between -MRtrix ``.mif`` files and the HDF5 format (``.h5``) used by ModelArray. This guide assumes -ModelArrayIO and MRtrix are already installed. +To convert fixel-wise data in MIF format to HDF5 format, +use the ``modelarrayio mif-to-h5`` command to convert the MIF files to the HDF5 format +(``.h5``) used by **ModelArray**, +and ``modelarrayio h5-to-mif`` to export results back to MIF. +This guide assumes **ModelArrayIO** and **MRtrix** are already installed. """ # %% @@ -61,17 +63,17 @@ # - age # - sex # * - FD -# - FD/sub-01_fd.mif +# - /home/username/myProject/data/FD/sub-01_fd.mif # - sub-01 # - 10 # - F # * - FD -# - FD/sub-02_fd.mif +# - /home/username/myProject/data/FD/sub-02_fd.mif # - sub-02 # - 20 # - M # * - FD -# - FD/sub-03_fd.mif +# - /home/username/myProject/data/FD/sub-03_fd.mif # - sub-03 # - 15 # - F @@ -112,7 +114,7 @@ # -------------------------------------- # # After running ModelArray and obtaining statistical results inside ``FD.h5`` (suppose the -# analysis name is ``"mylm"``), use ``fixelstats_write`` to export them as ``.mif`` files. +# analysis name is ``"mylm"``), use ``modelarrayio h5-to-mif`` to export them as ``.mif`` files. # The command also copies the original ``index.mif`` and ``directions.mif`` into the output # folder. # @@ -130,11 +132,11 @@ # # .. warning:: # -# **Existing files are not overwritten.** ``fixelstats_write`` calls ``mrconvert`` without +# **Existing files are not overwritten.** ``modelarrayio h5-to-mif`` calls ``mrconvert`` without # ``-force``, so any ``.mif`` file already present in ``--output-dir`` with the same name # will be left unchanged. If ``--output-dir`` itself already exists you will see a # ``WARNING: Output directory exists`` message, but no files will be deleted. To start -# fresh, manually remove the output directory before re-running ``fixelstats_write``. +# fresh, manually remove the output directory before re-running ``modelarrayio h5-to-mif``. # %% # Additional help diff --git a/docs/examples/plot_voxel_workflow.py b/docs/examples/plot_voxel_workflow.py index f4810c0..d9bfd8b 100644 --- a/docs/examples/plot_voxel_workflow.py +++ b/docs/examples/plot_voxel_workflow.py @@ -1,18 +1,19 @@ """ -Voxel-wise Data Conversion -========================== +NIfTI (Voxel-wise) Data Conversion +================================== -For voxel-wise data, use the **``convoxel``** command from **ModelArrayIO** to convert NIfTI -files to the HDF5 format (``.h5``) used by ModelArray, and ``volumestats_write`` to export -results back to NIfTI. The voxel workflow is very similar to the fixel workflow -(:ref:`sphx_glr_auto_examples_plot_fixel_workflow.py`). +For imaging data in NIfTI format, use the ``modelarrayio nifti-to-h5`` command to convert +the NIfTI files to the HDF5 format (``.h5``) used by **ModelArray**, +and ``modelarrayio h5-to-nifti`` to export results back to NIfTI. +The voxel workflow is very similar to the fixel workflow +(:ref:`sphx_glr_auto_examples_plot_mif_workflow.py`). """ # %% # Prepare data # ------------ # -# To convert a list of voxel-wise NIfTI files to ``.h5`` format, you need: +# To convert a list of NIfTI files to ``.h5`` format, you need: # # 1. **A cohort CSV** describing every NIfTI file to include (one CSV per scalar recommended). # 2. **A group mask** — only voxels inside the group mask are kept during conversion. @@ -22,7 +23,7 @@ # # Cohort CSV columns (names are fixed, not user-defined): # -# * ``scalar_name`` — which metric is being analysed (e.g. ``FA``) +# * ``scalar_name`` — which metric is being analysed (e.g., ``FA``) # * ``source_file`` — path to the subject's NIfTI file # * ``source_mask_file`` — path to the subject-specific mask (or the group mask if none exists) @@ -63,20 +64,20 @@ # - age # - sex # * - FA -# - FA/sub-01_FA.nii.gz -# - individual_masks/sub-01_mask.nii.gz +# - /home/username/myProject/data/FA/sub-01_FA.nii.gz +# - /home/username/myProject/data/individual_masks/sub-01_mask.nii.gz # - sub-01 # - 10 # - F # * - FA -# - FA/sub-02_FA.nii.gz -# - individual_masks/sub-02_mask.nii.gz +# - /home/username/myProject/data/FA/sub-02_FA.nii.gz +# - /home/username/myProject/data/individual_masks/sub-02_mask.nii.gz # - sub-02 # - 20 # - M # * - FA -# - FA/sub-03_FA.nii.gz -# - individual_masks/sub-03_mask.nii.gz +# - /home/username/myProject/data/FA/sub-03_FA.nii.gz +# - /home/username/myProject/data/individual_masks/sub-03_mask.nii.gz # - sub-03 # - 15 # - F @@ -95,7 +96,7 @@ # %% # Convert NIfTI files to HDF5 -# ---------------------------- +# --------------------------- # # Using the FA dataset from the example above: # @@ -114,10 +115,10 @@ # %% # Convert result .h5 back to NIfTI -# --------------------------------- +# -------------------------------- # -# After running ModelArray and obtaining statistical results inside ``FA.h5`` (suppose the -# analysis name is ``"mylm"``), use ``volumestats_write`` to export them as NIfTI files. +# After running **ModelArray** and obtaining statistical results inside ``FA.h5`` (suppose the +# analysis name is ``"mylm"``), use ``modelarrayio h5-to-nifti`` to export them as NIfTI files. # # .. code-block:: console # @@ -134,16 +135,15 @@ # # .. warning:: # -# If ``--output-dir`` already exists, ``volumestats_write`` will not delete it — you will +# If ``--output-dir`` already exists, ``modelarrayio h5-to-nifti`` will not delete it — you will # see ``WARNING: Output directory exists``. Existing files that are **not** part of the # current output list are left unchanged. Existing files that **are** part of the current -# output list will be overwritten (unlike the fixel ``fixelstats_write``, which does not -# overwrite via ``mrconvert``). To avoid confusion, consider manually deleting the output -# directory before re-running ``volumestats_write``. +# output list will be overwritten. To avoid confusion, consider manually deleting the output +# directory before re-running ``modelarrayio h5-to-nifti``. # %% # Number-of-observations image -# ----------------------------- +# ---------------------------- # # If you requested ``nobs`` during model fitting in ModelArray, after conversion you will find # an image called ``*_model.nobs.nii*``. With subject-specific masks, this image may be diff --git a/docs/index.rst b/docs/index.rst index a1d1114..1dfedb2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ .. toctree:: :maxdepth: 2 + installation auto_examples/index usage api diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..ad861f4 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,36 @@ +Installation +============ + +ModelArrayIO can be installed from pip. To install the latest official release: + +.. code-block:: bash + + pip install modelarrayio + +If you want to use the most up-to-date version, you can install from the ``main`` branch: + +.. code-block:: bash + + pip install git+https://github.com/PennLINC/ModelArrayIO.git + + +MRtrix (required for fixel ``.mif`` only) +----------------------------------------- + +For fixel-wise ``.mif`` conversion, the ``modelarrayio mif-to-h5`` / ``modelarrayio h5-to-mif`` tools use MRtrix ``mrconvert``. +Install MRtrix from `MRtrix's webpage `_ if needed. +Run ``mrview`` in the terminal to verify the installation. + +If your data are in NIfTI or CIFTI format only, you can skip this step. + + +What Next? +---------- + +For an overview of what you can do with ModelArrayIO see the `ModelArrayIO documentation `_. + +For an overview of what you can do with ModelArray see the `ModelArray documentation `_. + +To get right to using ModelArrayIO see the documentation on the `command line interface `_. + +If you have questions, or need help with using ModelArrayIO or ModelArray, check out `NeuroStars `_. diff --git a/src/modelarrayio/cli/cifti_to_h5.py b/src/modelarrayio/cli/cifti_to_h5.py index 63d80f2..6fec34b 100644 --- a/src/modelarrayio/cli/cifti_to_h5.py +++ b/src/modelarrayio/cli/cifti_to_h5.py @@ -13,14 +13,7 @@ from tqdm import tqdm from modelarrayio.cli import utils as cli_utils -from modelarrayio.cli.parser_utils import ( - add_backend_arg, - add_cohort_arg, - add_output_arg, - add_s3_workers_arg, - add_scalar_columns_arg, - add_storage_args, -) +from modelarrayio.cli.parser_utils import add_scalar_columns_arg, add_to_modelarray_args from modelarrayio.utils.cifti import ( _build_scalar_sources, _cohort_to_long_dataframe, @@ -35,7 +28,7 @@ def cifti_to_h5( cohort_file, backend='hdf5', - output='fixelarray.h5', + output=Path('greyordinatearray.h5'), storage_dtype='float32', compression='gzip', compression_level=4, @@ -43,8 +36,8 @@ def cifti_to_h5( chunk_voxels=0, target_chunk_mb=2.0, workers=None, - scalar_columns=None, s3_workers=1, + scalar_columns=None, ): """Load all CIFTI data and write to an HDF5 or TileDB file. @@ -73,10 +66,10 @@ def cifti_to_h5( workers : :obj:`int` Maximum number of parallel TileDB write workers (``None`` = auto). Has no effect when ``backend='hdf5'``. - scalar_columns : :obj:`list` - List of scalar columns to use s3_workers : :obj:`int` Number of workers for parallel S3 downloads + scalar_columns : :obj:`list` + List of scalar columns to use Returns ------- @@ -172,37 +165,11 @@ def _process_scalar_job(scalar_name, source_files): return 0 -def cifti_to_h5_main( - cohort_file, - backend='hdf5', - output='fixelarray.h5', - storage_dtype='float32', - compression='gzip', - compression_level=4, - shuffle=True, - chunk_voxels=0, - target_chunk_mb=2.0, - workers=None, - scalar_columns=None, - s3_workers=1, - log_level='INFO', -): +def cifti_to_h5_main(**kwargs): """Entry point for the ``modelarrayio cifti-to-h5`` command.""" + log_level = kwargs.pop('log_level', 'INFO') cli_utils.configure_logging(log_level) - return cifti_to_h5( - cohort_file=cohort_file, - backend=backend, - output=output, - storage_dtype=storage_dtype, - compression=compression, - compression_level=compression_level, - shuffle=shuffle, - chunk_voxels=chunk_voxels, - target_chunk_mb=target_chunk_mb, - workers=workers, - scalar_columns=scalar_columns, - s3_workers=s3_workers, - ) + return cifti_to_h5(**kwargs) def _parse_cifti_to_h5(): @@ -210,21 +177,6 @@ def _parse_cifti_to_h5(): description='Create a hdf5 file of CIFTI2 dscalar data', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - add_cohort_arg(parser) + add_to_modelarray_args(parser, default_output='greyordinatearray.h5') add_scalar_columns_arg(parser) - add_output_arg(parser, default_name='fixelarray.h5') - add_backend_arg(parser) - add_storage_args(parser) - parser.add_argument( - '--workers', - type=int, - help=( - 'Maximum number of parallel TileDB write workers. ' - 'Default 0 (auto, uses CPU count). ' - 'Set to 1 to disable parallel writes. ' - 'Has no effect when --backend=hdf5.' - ), - default=0, - ) - add_s3_workers_arg(parser) return parser diff --git a/src/modelarrayio/cli/h5_to_cifti.py b/src/modelarrayio/cli/h5_to_cifti.py index e2367e0..ffdde2a 100644 --- a/src/modelarrayio/cli/h5_to_cifti.py +++ b/src/modelarrayio/cli/h5_to_cifti.py @@ -12,7 +12,7 @@ import pandas as pd from modelarrayio.cli import utils as cli_utils -from modelarrayio.cli.parser_utils import _is_file, add_log_level_arg +from modelarrayio.cli.parser_utils import _is_file, add_from_modelarray_args, add_log_level_arg logger = logging.getLogger(__name__) @@ -117,28 +117,9 @@ def _parse_h5_to_cifti(): ) IsFile = partial(_is_file, parser=parser) - parser.add_argument( - '--analysis-name', - '--analysis_name', - help='Name for the statistical analysis results to be saved.', - ) - parser.add_argument( - '--input-hdf5', - '--input_hdf5', - help='Name of HDF5 (.h5) file where results outputs are saved.', - type=IsFile, - dest='in_file', - ) - parser.add_argument( - '--output-dir', - '--output_dir', - help=( - 'Directory where outputs will be saved. ' - 'If the directory does not exist, it will be automatically created.' - ), - ) + add_from_modelarray_args(parser) - example_cifti_group = parser.add_mutually_exclusive_group() + example_cifti_group = parser.add_mutually_exclusive_group(required=True) example_cifti_group.add_argument( '--cohort-file', '--cohort_file', @@ -147,14 +128,12 @@ def _parse_h5_to_cifti(): 'Used to select an example CIFTI file if no example CIFTI file is provided.' ), type=IsFile, - required=False, default=None, ) example_cifti_group.add_argument( '--example-cifti', '--example_cifti', help='Path to an example cifti file.', - required=False, type=IsFile, default=None, ) diff --git a/src/modelarrayio/cli/h5_to_mif.py b/src/modelarrayio/cli/h5_to_mif.py index 812f95f..bb6bfed 100644 --- a/src/modelarrayio/cli/h5_to_mif.py +++ b/src/modelarrayio/cli/h5_to_mif.py @@ -13,7 +13,7 @@ import pandas as pd from modelarrayio.cli import utils as cli_utils -from modelarrayio.cli.parser_utils import _is_file, add_log_level_arg +from modelarrayio.cli.parser_utils import _is_file, add_from_modelarray_args, add_log_level_arg from modelarrayio.utils.fixels import mif_to_nifti2, nifti2_to_mif logger = logging.getLogger(__name__) @@ -86,10 +86,11 @@ def h5_to_mif(example_mif, in_file, analysis_name, output_dir): def h5_to_mif_main( index_file, directions_file, - cohort_file, analysis_name, in_file, output_dir, + cohort_file=None, + example_mif=None, log_level='INFO', ): """Entry point for the ``modelarrayio h5-to-mif`` command.""" @@ -105,8 +106,13 @@ def h5_to_mif_main( output_path / Path(index_file).name, ) - cohort_df = pd.read_csv(cohort_file) - example_mif = cohort_df['source_file'].iloc[0] + if example_mif is None: + logger.warning( + 'No example MIF file provided, using the first MIF file from the cohort file' + ) + cohort_df = pd.read_csv(cohort_file) + example_mif = cohort_df['source_file'].iloc[0] + h5_to_mif( example_mif=example_mif, in_file=in_file, @@ -126,43 +132,37 @@ def _parse_h5_to_mif(): parser.add_argument( '--index-file', '--index_file', - help='Index File', + help='Index file used to reconstruct MIF files.', required=True, type=IsFile, ) parser.add_argument( '--directions-file', '--directions_file', - help='Directions File', + help='Directions file used to reconstruct MIF files.', required=True, type=IsFile, ) - parser.add_argument( + + add_from_modelarray_args(parser) + + example_mif_group = parser.add_mutually_exclusive_group(required=True) + example_mif_group.add_argument( '--cohort-file', '--cohort_file', - help='Path to a csv with demographic info and paths to data.', - required=True, + help=( + 'Path to a csv with demographic info and paths to data. ' + 'Used to select an example MIF file if no example MIF file is provided.' + ), type=IsFile, + default=None, ) - parser.add_argument( - '--analysis-name', - '--analysis_name', - help='Name for the statistical analysis results to be saved.', - ) - parser.add_argument( - '--input-hdf5', - '--input_hdf5', - help='Name of HDF5 (.h5) file where results outputs are saved.', + example_mif_group.add_argument( + '--example-mif', + '--example_mif', + help='Path to an example MIF file.', type=IsFile, - dest='in_file', - ) - parser.add_argument( - '--output-dir', - '--output_dir', - help=( - 'Fixel directory where outputs will be saved. ' - 'If the directory does not exist, it will be automatically created.' - ), + default=None, ) add_log_level_arg(parser) return parser diff --git a/src/modelarrayio/cli/h5_to_nifti.py b/src/modelarrayio/cli/h5_to_nifti.py index 2c11295..4ff6ca9 100644 --- a/src/modelarrayio/cli/h5_to_nifti.py +++ b/src/modelarrayio/cli/h5_to_nifti.py @@ -12,7 +12,7 @@ import numpy as np from modelarrayio.cli import utils as cli_utils -from modelarrayio.cli.parser_utils import _is_file, add_log_level_arg +from modelarrayio.cli.parser_utils import _is_file, add_from_modelarray_args, add_log_level_arg logger = logging.getLogger(__name__) @@ -28,9 +28,8 @@ def h5_to_nifti(in_file, analysis_name, group_mask_file, output_extension, outpu # modify the header: header_tosave = group_mask_img.header - header_tosave.set_data_dtype( - data_type_tosave - ) # modify the data type (mask's data type could be uint8...) + # modify the data type (mask's data type could be uint8...) + header_tosave.set_data_dtype(data_type_tosave) output_path = Path(output_dir) with h5py.File(in_file, 'r') as h5_data: @@ -100,26 +99,9 @@ def _parse_h5_to_nifti(): required=True, type=IsFile, ) - parser.add_argument( - '--analysis-name', - '--analysis_name', - help='Name of the statistical analysis results to be saved.', - ) - parser.add_argument( - '--input-hdf5', - '--input_hdf5', - help='Name of HDF5 (.h5) file where results outputs are saved.', - type=IsFile, - dest='in_file', - ) - parser.add_argument( - '--output-dir', - '--output_dir', - help=( - 'A directory where output volume files will be saved. ' - 'If the directory does not exist, it will be automatically created.' - ), - ) + + add_from_modelarray_args(parser) + parser.add_argument( '--output-ext', '--output_ext', diff --git a/src/modelarrayio/cli/mif_to_h5.py b/src/modelarrayio/cli/mif_to_h5.py index 2933f89..38ed1cc 100644 --- a/src/modelarrayio/cli/mif_to_h5.py +++ b/src/modelarrayio/cli/mif_to_h5.py @@ -13,13 +13,7 @@ from tqdm import tqdm from modelarrayio.cli import utils as cli_utils -from modelarrayio.cli.parser_utils import ( - _is_file, - add_backend_arg, - add_cohort_arg, - add_output_arg, - add_storage_args, -) +from modelarrayio.cli.parser_utils import _is_file, add_to_modelarray_args from modelarrayio.utils.fixels import gather_fixels, mif_to_nifti2 logger = logging.getLogger(__name__) @@ -37,6 +31,8 @@ def mif_to_h5( shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, + workers=None, + s3_workers=1, ): """Load all fixeldb data and write to an HDF5 or TileDB file. @@ -66,6 +62,11 @@ def mif_to_h5( Chunk/tile size along the fixel axis (0 = auto) target_chunk_mb : :obj:`float` Target chunk/tile size in MiB when auto-computing the spatial axis length + workers : :obj:`int` + Maximum number of parallel TileDB write workers. Default 0 (auto). + Has no effect when ``backend='hdf5'``. + s3_workers : :obj:`int` + Number of parallel workers for S3 downloads. Default 1. Returns ------- @@ -122,35 +123,11 @@ def mif_to_h5( return 0 -def mif_to_h5_main( - index_file, - directions_file, - cohort_file, - backend='hdf5', - output='fixelarray.h5', - storage_dtype='float32', - compression='gzip', - compression_level=4, - shuffle=True, - chunk_voxels=0, - target_chunk_mb=2.0, - log_level='INFO', -): +def mif_to_h5_main(**kwargs): """Entry point for the ``modelarrayio mif-to-h5`` command.""" + log_level = kwargs.pop('log_level', 'INFO') cli_utils.configure_logging(log_level) - return mif_to_h5( - index_file=index_file, - directions_file=directions_file, - cohort_file=cohort_file, - backend=backend, - output=output, - storage_dtype=storage_dtype, - compression=compression, - compression_level=compression_level, - shuffle=shuffle, - chunk_voxels=chunk_voxels, - target_chunk_mb=target_chunk_mb, - ) + return mif_to_h5(**kwargs) def _parse_mif_to_h5(): @@ -160,6 +137,7 @@ def _parse_mif_to_h5(): ) IsFile = partial(_is_file, parser=parser) + # MIF-specific arguments parser.add_argument( '--index-file', '--index_file', @@ -174,8 +152,7 @@ def _parse_mif_to_h5(): required=True, type=IsFile, ) - add_cohort_arg(parser) - add_output_arg(parser, default_name='fixelarray.h5') - add_backend_arg(parser) - add_storage_args(parser) + + # Common arguments + add_to_modelarray_args(parser, default_output='fixelarray.h5') return parser diff --git a/src/modelarrayio/cli/nifti_to_h5.py b/src/modelarrayio/cli/nifti_to_h5.py index d35c17b..28e2422 100644 --- a/src/modelarrayio/cli/nifti_to_h5.py +++ b/src/modelarrayio/cli/nifti_to_h5.py @@ -13,14 +13,7 @@ import pandas as pd from modelarrayio.cli import utils as cli_utils -from modelarrayio.cli.parser_utils import ( - _is_file, - add_backend_arg, - add_cohort_arg, - add_output_arg, - add_s3_workers_arg, - add_storage_args, -) +from modelarrayio.cli.parser_utils import _is_file, add_to_modelarray_args from modelarrayio.utils.voxels import _load_cohort_voxels logger = logging.getLogger(__name__) @@ -30,13 +23,14 @@ def nifti_to_h5( group_mask_file, cohort_file, backend='hdf5', - output='voxeldb.h5', + output=Path('voxelarray.h5'), storage_dtype='float32', compression='gzip', compression_level=4, shuffle=True, chunk_voxels=0, target_chunk_mb=2.0, + workers=None, s3_workers=1, ): """Load all volume data and write to an HDF5 or TileDB file. @@ -65,6 +59,9 @@ def nifti_to_h5( Chunk/tile size along the voxel axis. If 0, auto-compute. Default 0. target_chunk_mb : :obj:`float` Target chunk/tile size in MiB when auto-computing. Default 2.0. + workers : :obj:`int` + Maximum number of parallel TileDB write workers. Default 0 (auto). + Has no effect when ``backend='hdf5'``. s3_workers : :obj:`int` Number of parallel workers for S3 downloads. Default 1. """ @@ -118,35 +115,11 @@ def nifti_to_h5( return 0 -def nifti_to_h5_main( - group_mask_file, - cohort_file, - backend='hdf5', - output='voxeldb.h5', - storage_dtype='float32', - compression='gzip', - compression_level=4, - shuffle=True, - chunk_voxels=0, - target_chunk_mb=2.0, - s3_workers=1, - log_level='INFO', -): +def nifti_to_h5_main(**kwargs): """Entry point for the ``modelarrayio nifti-to-h5`` command.""" + log_level = kwargs.pop('log_level', 'INFO') cli_utils.configure_logging(log_level) - return nifti_to_h5( - group_mask_file=group_mask_file, - cohort_file=cohort_file, - backend=backend, - output=output, - storage_dtype=storage_dtype, - compression=compression, - compression_level=compression_level, - shuffle=shuffle, - chunk_voxels=chunk_voxels, - target_chunk_mb=target_chunk_mb, - s3_workers=s3_workers, - ) + return nifti_to_h5(**kwargs) def _parse_nifti_to_h5(): @@ -155,6 +128,8 @@ def _parse_nifti_to_h5(): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) IsFile = partial(_is_file, parser=parser) + + # NIfTI-specific arguments parser.add_argument( '--group-mask-file', '--group_mask_file', @@ -162,9 +137,7 @@ def _parse_nifti_to_h5(): required=True, type=IsFile, ) - add_cohort_arg(parser) - add_output_arg(parser, default_name='voxeldb.h5') - add_backend_arg(parser) - add_storage_args(parser) - add_s3_workers_arg(parser) + + # Common arguments + add_to_modelarray_args(parser, default_output='voxelarray.h5') return parser diff --git a/src/modelarrayio/cli/parser_utils.py b/src/modelarrayio/cli/parser_utils.py index 87f7a29..55c2c4a 100644 --- a/src/modelarrayio/cli/parser_utils.py +++ b/src/modelarrayio/cli/parser_utils.py @@ -4,30 +4,29 @@ from pathlib import Path -def add_output_arg(parser, default_name='output.h5'): +def add_to_modelarray_args(parser, default_output='output.h5'): + """Add arguments common to all commands that prepare data for ModelArray.""" + parser.add_argument( + '--cohort-file', + '--cohort_file', + help='Path to a csv with demographic info and paths to data.', + required=True, + type=partial(_is_file, parser=parser), + ) parser.add_argument( '--output', help=( 'Output path. For the hdf5 backend, path to an .h5 file; ' 'for the tiledb backend, path to a .tdb directory.' ), - default=default_name, + default=default_output, ) - return parser - - -def add_cohort_arg(parser): parser.add_argument( - '--cohort-file', - '--cohort_file', - help='Path to a csv with demographic info and paths to data.', - required=True, - type=partial(_is_file, parser=parser), + '--backend', + help='Storage backend for subject-by-element matrix', + choices=['hdf5', 'tiledb'], + default='hdf5', ) - return parser - - -def add_storage_args(parser): parser.add_argument( '--dtype', help='Floating dtype for storing values: float32 (default) or float64', @@ -80,22 +79,21 @@ def add_storage_args(parser): default=2.0, ) - add_log_level_arg(parser) - return parser - - -def add_backend_arg(parser): - parser.add_argument( - '--backend', - help='Storage backend for subject-by-element matrix', - choices=['hdf5', 'tiledb'], - default='hdf5', + tiledb_group = parser.add_argument_group('TileDB arguments') + tiledb_group.add_argument( + '--workers', + type=int, + help=( + 'Maximum number of parallel TileDB write workers. ' + 'Default 0 (auto, uses CPU count). ' + 'Set to 1 to disable parallel writes. ' + 'Has no effect when --backend=hdf5.' + ), + default=0, ) - return parser - -def add_s3_workers_arg(parser): - parser.add_argument( + s3_group = parser.add_argument_group('S3 arguments') + s3_group.add_argument( '--s3-workers', '--s3_workers', type=int, @@ -106,6 +104,9 @@ def add_s3_workers_arg(parser): 'Default 1 (serial).' ), ) + + add_log_level_arg(parser) + return parser @@ -134,6 +135,34 @@ def add_log_level_arg(parser): return parser +def add_from_modelarray_args(parser): + parser.add_argument( + '--analysis-name', + '--analysis_name', + help='Name for the statistical analysis results to be saved.', + required=True, + ) + parser.add_argument( + '--input-hdf5', + '--input_hdf5', + help='Name of HDF5 (.h5) file where results outputs are saved.', + type=partial(_is_file, parser=parser), + dest='in_file', + required=True, + ) + parser.add_argument( + '--output-dir', + '--output_dir', + help=( + 'Directory where outputs will be saved. ' + 'If the directory does not exist, it will be automatically created.' + ), + required=True, + ) + + return parser + + def _path_exists(path: str | Path | None, parser) -> Path: """Ensure a given path exists.""" if path is None or not Path(path).exists(): diff --git a/test/data_voxel_toy/README.md b/test/data_voxel_toy/README.md index 25ff1be..bc505ca 100644 --- a/test/data_voxel_toy/README.md +++ b/test/data_voxel_toy/README.md @@ -1,4 +1,5 @@ -This is a toy voxel-wise dataset for testing ModelArrayIO’s voxel pipeline (`convoxel` / `volumestats_write`). +This is a toy voxel-wise dataset for testing ModelArrayIO’s voxel pipeline (`modelarrayio nifti-to-h5` / `modelarrayio h5-to-nifti`). + * It was generated using scripts in [ModelArray_tests](https://github.com/PennLINC/ModelArray_tests) GitHub repository. * Using scripts in folder `testdata_ConFixel` there (historical folder name from the ConFixel era). * It includes n=20 voxel-wise data with individual masks and a group mask. diff --git a/test/test_parser_utils.py b/test/test_parser_utils.py index c08a7c6..4aed56a 100644 --- a/test/test_parser_utils.py +++ b/test/test_parser_utils.py @@ -9,11 +9,7 @@ def _parser_with_cohort() -> argparse.ArgumentParser: p = argparse.ArgumentParser() - parser_utils.add_output_arg(p) - parser_utils.add_cohort_arg(p) - parser_utils.add_storage_args(p) - parser_utils.add_backend_arg(p) - parser_utils.add_s3_workers_arg(p) + parser_utils.add_to_modelarray_args(p) return p @@ -63,30 +59,25 @@ def test_storage_aliases_and_no_shuffle(tmp_path_factory) -> None: assert args.log_level == 'WARNING' -def test_output_hdf5_default_name_override() -> None: +def test_output_hdf5_default_name_override(tmp_path_factory) -> None: + tmp_path = tmp_path_factory.mktemp('test_output_hdf5_default_name_override') + cohort_file = tmp_path / 'cohort.csv' + cohort_file.write_text('subject_id,path\n1,path1\n2,path2') p = argparse.ArgumentParser() - parser_utils.add_output_arg(p, default_name='custom.h5') - args = p.parse_args([]) + parser_utils.add_to_modelarray_args(p, default_output='custom.h5') + args = p.parse_args(['--cohort-file', str(cohort_file)]) assert args.output == 'custom.h5' -def test_tiledb_args_group() -> None: - p = argparse.ArgumentParser() - parser_utils.add_output_arg(p, default_name='arrays.tdb') - parser_utils.add_storage_args(p) - args = p.parse_args([]) - assert args.output == 'arrays.tdb' - assert args.compression == 'gzip' - assert args.chunk_voxels == 0 - - -def test_scalar_columns_optional(tmp_path_factory) -> None: - tmp_path = tmp_path_factory.mktemp('test_scalar_columns_optional') +def test_tiledb_args_group(tmp_path_factory) -> None: + tmp_path = tmp_path_factory.mktemp('test_tiledb_args_group') cohort_file = tmp_path / 'cohort.csv' cohort_file.write_text('subject_id,path\n1,path1\n2,path2') - p = argparse.ArgumentParser() - parser_utils.add_cohort_arg(p) - parser_utils.add_scalar_columns_arg(p) - args = p.parse_args(['--cohort-file', str(cohort_file), '--scalar-columns', 'c1', 'c2']) - assert args.scalar_columns == ['c1', 'c2'] + parser_utils.add_to_modelarray_args(p, default_output='arrays.tdb') + args = p.parse_args(['--cohort-file', str(cohort_file), '--backend', 'tiledb']) + assert args.output == 'arrays.tdb' + assert args.backend == 'tiledb' + assert args.workers == 0 + assert args.s3_workers == 1 + assert args.log_level == 'INFO'