diff --git a/.circleci/config.yml b/.circleci/config.yml index 56f7150..9cc8c57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,11 +69,10 @@ jobs: -v /tmp/data:/inputs \ -v /tmp:/tmp \ pennbbl/fixeldb:latest \ - --relative-root /inputs \ - --directions-file fixeldata/directions.mif \ - --index-file fixeldata/index.mif \ - --cohort-file test_cohort.csv \ - --output-hdf5 fixels.h5 + --directions-file /inputs/fixeldata/directions.mif \ + --index-file /inputs/fixeldata/index.mif \ + --cohort-file /inputs/test_cohort.csv \ + --output-hdf5 /inputs/fixels.h5 - store_artifacts: path: /tmp/data/fixels.h5 diff --git a/.gitignore b/.gitignore index 3bc9ae7..b429e48 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,7 @@ instance/ # Sphinx documentation docs/_build/ docs/generated/ +docs/auto_examples/ # PyBuilder target/ diff --git a/docs/conf.py b/docs/conf.py index ee1eb27..e3e71ae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,8 +24,18 @@ 'sphinxarg.ext', 'sphinx_copybutton', 'sphinx_rtd_theme', + 'sphinx_gallery.gen_gallery', ] +sphinx_gallery_conf = { + 'examples_dirs': ['examples'], + 'gallery_dirs': ['auto_examples'], + 'filename_pattern': r'/plot_', + 'abort_on_example_error': False, + 'download_all_examples': False, + 'plot_gallery': False, +} + templates_path = ['_templates'] source_suffix = {'.rst': 'restructuredtext'} diff --git a/docs/examples/README.rst b/docs/examples/README.rst new file mode 100644 index 0000000..792b21c --- /dev/null +++ b/docs/examples/README.rst @@ -0,0 +1,5 @@ +Walkthroughs +============ + +Step-by-step guides for converting neuroimaging data to and from the HDF5 +format used by `ModelArray `_. diff --git a/docs/examples/plot_fixel_workflow.py b/docs/examples/plot_fixel_workflow.py new file mode 100644 index 0000000..cd94d88 --- /dev/null +++ b/docs/examples/plot_fixel_workflow.py @@ -0,0 +1,150 @@ +""" +Fixel-wise Data Conversion +========================== + +For fixel-wise data, use the **``confixel``** command from **ModelArrayIO** to convert between +MRtrix ``.mif`` files and the HDF5 format (``.h5``) used by ModelArray. This guide assumes +ModelArrayIO and MRtrix are already installed. +""" + +# %% +# Prepare data +# ------------ +# +# To convert a list of fixel-wise data from ``.mif`` to ``.h5`` format, prepare a cohort CSV +# file that describes every ``.mif`` file you want to include. We recommend one CSV per scalar +# (e.g. FD, FC, FDC), yielding one ``.h5`` file per scalar. +# +# Cohort CSV columns (names are fixed, not user-defined): +# +# * ``scalar_name`` — which metric is being analysed (e.g. ``FD``, ``FC``, ``FDC``) +# * ``source_file`` — path to the ``.mif`` file for this subject + +# %% +# Example folder structure +# ------------------------ +# +# .. code-block:: text +# +# /home/username/myProject/data +# | +# ├── cohort_FD.csv +# │ +# ├── FD +# │ ├── index.mif +# │ ├── directions.mif +# │ ├── sub-01_fd.mif +# │ ├── sub-02_fd.mif +# │ ├── sub-03_fd.mif +# │ └── ... +# │ +# ├── FC +# │ ├── index.mif +# │ ├── directions.mif +# │ ├── sub-01_fc.mif +# │ ├── sub-02_fc.mif +# │ ├── sub-03_fc.mif +# │ └── ... +# └── ... +# +# These ``.mif`` files are generated by MRtrix. +# +# Corresponding ``cohort_FD.csv`` for scalar FD: +# +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - **scalar_name** *(required)* +# - **source_file** *(required)* +# - subject_id +# - age +# - sex +# * - FD +# - FD/sub-01_fd.mif +# - sub-01 +# - 10 +# - F +# * - FD +# - FD/sub-02_fd.mif +# - sub-02 +# - 20 +# - M +# * - FD +# - FD/sub-03_fd.mif +# - sub-03 +# - 15 +# - F +# * - ... +# - ... +# - ... +# - ... +# - ... +# +# Notes: +# +# * Column order does not matter. +# * Values are case-sensitive — folder names, file names, and scalar names must match exactly +# between the CSV and disk. + +# %% +# Convert .mif files to HDF5 +# -------------------------- +# +# Using the FD dataset from the example above: +# +# .. code-block:: console +# +# # activate your conda environment first +# conda activate +# +# confixel \ +# --index-file /home/username/myProject/data/FD/index.mif \ +# --directions-file /home/username/myProject/data/FD/directions.mif \ +# --cohort-file /home/username/myProject/data/cohort_FD.csv \ +# --output-hdf5 /home/username/myProject/data/FD.h5 +# +# This produces ``FD.h5`` in ``/home/username/myProject/data``. You can then use +# `ModelArray `_ to run statistical analyses on it. + +# %% +# Convert result .h5 back to .mif files +# -------------------------------------- +# +# After running ModelArray and obtaining statistical results inside ``FD.h5`` (suppose the +# analysis name is ``"mylm"``), use ``fixelstats_write`` to export them as ``.mif`` files. +# The command also copies the original ``index.mif`` and ``directions.mif`` into the output +# folder. +# +# .. code-block:: console +# +# fixelstats_write \ +# --index-file /home/username/myProject/data/FD/index.mif \ +# --directions-file /home/username/myProject/data/FD/directions.mif \ +# --cohort-file /home/username/myProject/data/cohort_FD.csv \ +# --analysis-name mylm \ +# --input-hdf5 /home/username/myProject/data/FD.h5 \ +# --output-dir /home/username/myProject/data/FD_stats +# +# The results in ``FD_stats`` can now be viewed in ``mrview``. +# +# .. warning:: +# +# **Existing files are not overwritten.** ``fixelstats_write`` calls ``mrconvert`` without +# ``-force``, so any ``.mif`` file already present in ``--output-dir`` with the same name +# will be left unchanged. If ``--output-dir`` itself already exists you will see a +# ``WARNING: Output directory exists`` message, but no files will be deleted. To start +# fresh, manually remove the output directory before re-running ``fixelstats_write``. + +# %% +# Additional help +# --------------- +# +# Full argument documentation is available from the command line: +# +# .. code-block:: console +# +# confixel --help +# fixelstats_write --help +# +# or in the :doc:`/usage` page of this documentation. diff --git a/docs/examples/plot_voxel_workflow.py b/docs/examples/plot_voxel_workflow.py new file mode 100644 index 0000000..87a24f0 --- /dev/null +++ b/docs/examples/plot_voxel_workflow.py @@ -0,0 +1,183 @@ +""" +Voxel-wise Data Conversion +========================== + +For voxel-wise data, use the **``convoxel``** command from **ModelArrayIO** to convert NIfTI +files to the HDF5 format (``.h5``) used by ModelArray, and ``volumestats_write`` to export +results back to NIfTI. The voxel workflow is very similar to the fixel workflow +(:ref:`sphx_glr_auto_examples_plot_fixel_workflow.py`). +""" + +# %% +# Prepare data +# ------------ +# +# To convert a list of voxel-wise NIfTI files to ``.h5`` format, you need: +# +# 1. **A cohort CSV** describing every NIfTI file to include (one CSV per scalar recommended). +# 2. **A group mask** — only voxels inside the group mask are kept during conversion. +# 3. **Subject-specific masks** *(optional)* — voxels outside each subject's mask are set to +# ``NaN`` after conversion. If you do not have per-subject masks, supply the group mask for +# every subject (see the CSV example below). +# +# Cohort CSV columns (names are fixed, not user-defined): +# +# * ``scalar_name`` — which metric is being analysed (e.g. ``FA``) +# * ``source_file`` — path to the subject's NIfTI file +# * ``source_mask_file`` — path to the subject-specific mask (or the group mask if none exists) + +# %% +# Example folder structure +# ------------------------ +# +# .. code-block:: text +# +# /home/username/myProject/data +# | +# ├── cohort_FA.csv +# ├── group_mask.nii.gz +# │ +# ├── FA +# │ ├── sub-01_FA.nii.gz +# │ ├── sub-02_FA.nii.gz +# │ ├── sub-03_FA.nii.gz +# │ └── ... +# │ +# ├── individual_masks +# │ ├── sub-01_mask.nii.gz +# │ ├── sub-02_mask.nii.gz +# │ ├── sub-03_mask.nii.gz +# │ └── ... +# └── ... +# +# Corresponding ``cohort_FA.csv`` for scalar FA: +# +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - **scalar_name** *(required)* +# - **source_file** *(required)* +# - **source_mask_file** *(required)* +# - subject_id +# - age +# - sex +# * - FA +# - FA/sub-01_FA.nii.gz +# - individual_masks/sub-01_mask.nii.gz +# - sub-01 +# - 10 +# - F +# * - FA +# - FA/sub-02_FA.nii.gz +# - individual_masks/sub-02_mask.nii.gz +# - sub-02 +# - 20 +# - M +# * - FA +# - FA/sub-03_FA.nii.gz +# - individual_masks/sub-03_mask.nii.gz +# - sub-03 +# - 15 +# - F +# * - ... +# - ... +# - ... +# - ... +# - ... +# - ... +# +# Notes: +# +# * Column order does not matter. +# * Values are case-sensitive — folder names, file names, and scalar names must match exactly +# between the CSV and disk. + +# %% +# Convert NIfTI files to HDF5 +# ---------------------------- +# +# Using the FA dataset from the example above: +# +# .. code-block:: console +# +# # activate your conda environment first +# conda activate +# +# convoxel \ +# --group-mask-file /home/username/myProject/data/group_mask.nii.gz \ +# --cohort-file /home/username/myProject/data/cohort_FA.csv \ +# --output-hdf5 /home/username/myProject/data/FA.h5 +# +# This produces ``FA.h5`` in ``/home/username/myProject/data``. You can then use +# `ModelArray `_ to run statistical analyses on it. + +# %% +# Convert result .h5 back to NIfTI +# --------------------------------- +# +# After running ModelArray and obtaining statistical results inside ``FA.h5`` (suppose the +# analysis name is ``"mylm"``), use ``volumestats_write`` to export them as NIfTI files. +# +# .. code-block:: console +# +# volumestats_write \ +# --group-mask-file /home/username/myProject/data/group_mask.nii.gz \ +# --cohort-file /home/username/myProject/data/cohort_FA.csv \ +# --analysis-name mylm \ +# --input-hdf5 /home/username/myProject/data/FA.h5 \ +# --output-dir /home/username/myProject/data/FA_stats \ +# --output-ext .nii.gz +# +# All converted volume data are saved as ``float32``. Results in ``FA_stats`` can be viewed +# with any NIfTI image viewer. +# +# .. warning:: +# +# If ``--output-dir`` already exists, ``volumestats_write`` will not delete it — you will +# see ``WARNING: Output directory exists``. Existing files that are **not** part of the +# current output list are left unchanged. Existing files that **are** part of the current +# output list will be overwritten (unlike the fixel ``fixelstats_write``, which does not +# overwrite via ``mrconvert``). To avoid confusion, consider manually deleting the output +# directory before re-running ``volumestats_write``. + +# %% +# Number-of-observations image +# ----------------------------- +# +# If you requested ``nobs`` during model fitting in ModelArray, after conversion you will find +# an image called ``*_model.nobs.nii*``. With subject-specific masks, this image may be +# inhomogeneous across voxels. +# +# Voxels that did not have sufficient subjects (due to subject-specific masking) are stored as +# ``NaN`` in the HDF5 file. How different viewers display these voxels: +# +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - Viewer +# - Regular voxel +# - Voxel without sufficient subjects +# * - nibabel (Python) +# - e.g. ``nobs: 209.0``, ``p.value: 0.01`` +# - ``NaN`` +# * - MRtrix mrview +# - e.g. ``nobs: 209``, ``p.value: 0.01`` +# - ``?`` +# * - ITK-SNAP +# - e.g. ``nobs: 209``, ``p.value: 0.01`` +# - ``0`` (displayed, but excluded when thresholding) + +# %% +# Additional help +# --------------- +# +# Full argument documentation is available from the command line: +# +# .. code-block:: console +# +# convoxel --help +# volumestats_write --help +# +# or in the :doc:`/usage` page of this documentation. diff --git a/docs/index.rst b/docs/index.rst index 4efddc6..a1d1114 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,5 +11,6 @@ .. toctree:: :maxdepth: 2 + auto_examples/index usage api diff --git a/docs/usage.rst b/docs/usage.rst index f367abe..a2e27fc 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -8,7 +8,7 @@ confixel ******** .. argparse:: - :ref: modelarrayio.cli.fixels_to_h5.get_parser + :ref: modelarrayio.cli.mif_to_h5.get_parser :prog: confixel :func: get_parser @@ -18,7 +18,7 @@ convoxel ******** .. argparse:: - :ref: modelarrayio.cli.voxels_to_h5.get_parser + :ref: modelarrayio.cli.nifti_to_h5.get_parser :prog: convoxel :func: get_parser @@ -38,7 +38,7 @@ fixelstats_write **************** .. argparse:: - :ref: modelarrayio.cli.h5_to_fixels.get_parser + :ref: modelarrayio.cli.h5_to_mif.get_parser :prog: fixelstats_write :func: get_parser @@ -47,7 +47,7 @@ volumestats_write ***************** .. argparse:: - :ref: modelarrayio.cli.h5_to_voxels.get_parser + :ref: modelarrayio.cli.h5_to_nifti.get_parser :prog: volumestats_write :func: get_parser diff --git a/notebooks/README.md b/notebooks/README.md deleted file mode 100644 index 789ad28..0000000 --- a/notebooks/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Walkthrough for different imaging data types - -* [Walkthrough for fixel-wise data conversion](walkthrough_fixel-wise_data.md) (`confixel` / `fixelstats_write` in ModelArrayIO) -* [Walkthrough for voxel-wise data conversion](walkthrough_voxel-wise_data.md) (`convoxel` / `volumestats_write` in ModelArrayIO) diff --git a/notebooks/walkthrough_fixel-wise_data.md b/notebooks/walkthrough_fixel-wise_data.md deleted file mode 100644 index 6cfc6a6..0000000 --- a/notebooks/walkthrough_fixel-wise_data.md +++ /dev/null @@ -1,109 +0,0 @@ -# Walkthrough for fixel-wise data conversion - -For fixel-wise data, use the **`confixel`** command (from the **ModelArrayIO** Python package) to convert between MRtrix `.mif` files and the HDF5 file format (`.h5`) that ModelArray uses. This guide assumes you followed the [installation guide](../README.rst#installation), installed **ModelArrayIO**, and installed **MRtrix** for fixel-wise workflows. - -## Prepare data - -To convert (a list of) fixel-wise data from .mif format to .h5 format, you need to prepare a cohort CSV file that provides basic information for all .mif files you want to include. We recommend that, for each scalar (e.g. FD/FC/FDC), prepare one .csv file, and thus getting one .h5 file. - -### Cohort's csv file (for each scalar) - -Each row of a cohort .csv is for one mif file you want to include. The file should at least include these columns (Notes: these column names are fixed, i.e. not user-defined): - -* "scalar_name": which tells us what metric is being analyzed, e.g. FD, FC, or FDC -* "source_file": which tells us which mif file will be used for this subject - -### Example -#### Example **folder structure**: - -``` -/home/username/myProject/data -| -├── cohort_FD.csv -│ -├── FD -│ ├── index.mif -│ ├── directions.mif -| ├── sub-01_fd.mif -| ├── sub-02_fd.mif -| ├── sub-03_fd.mif -│ ├── ... -│ -├── FC -│ ├── index.mif -│ ├── directions.mif -| ├── sub-01_fc.mif -| ├── sub-02_fc.mif -| ├── sub-03_fc.mif -| ├── ... -└── ... -``` -These mif files are generated by `MRtrix`. - -#### Corresponding **csv file for scalar FD** can look like this: -"cohort_FD.csv": -| ***scalar_name*** | ***source_file*** | subject_id | age | sex | -| :----: | :----: | :----: | :----: | :----: | -| FD | FD/sub-01_fd.mif | sub-01 | 10 | F | -| FD | FD/sub-02_fd.mif | sub-02 | 20 | M | -| FD | FD/sub-03_fd.mif | sub-03 | 15 | F | -| ... | ... | ... | ... | ... | - -Notes: -* Columns that must be included are highlighted in ***bold and italics***; -* The order of columns can be changed. -* Please make sure the consistency (see examples as below). These strings case sensitive, too! Either upper case or lower case works, just need to be consistent :) - * Folder name e.g., "FD" in column `source_file` in CSV file v.s. the actual folder name on disk; - * File names in column `source_file` in CSV file v.s. the actual file names on disk; - * Scalar name e.g., "FD" in column `scalar_name` in CSV file v.s. what you will specify when using functions in `ModelArray`; - -For this case, when running `confixel` to create HDF5 fixel-wise data, argument **--relative-root** should be "/home/username/myProject/data" - - -## Run `confixel` and `fixelstats_write` -### Convert .mif files to an HDF5 (.h5) file -Using above described scenario as an example, for FD dataset: -``` console -# first, activate conda environment with `conda activate ` -confixel \ - --index-file FD/index.mif \ - --directions-file FD/directions.mif \ - --cohort-file cohort_FD.csv \ - --relative-root /home/username/myProject/data \ - --output-hdf5 FD.h5 -``` - -Now you should get the HDF5 file "FD.h5" in folder "/home/username/myProject/data". You may use [ModelArray](https://pennlinc.github.io/ModelArray/) to perform statistical analysis. - -### Convert result .h5 file to .mif files: -After running `ModelArray` and getting statistical results in FD.h5 file (say, the analysis name is called "mylm"), you can use `fixelstats_write` to convert results into a list of .mif files in a folder specified by you. This command will also copy the original index.mif and directions.mif to this folder. -``` console -# first, activate conda environment with `conda activate ` -fixelstats_write \ - --index-file FD/index.mif \ - --directions-file FD/directions.mif \ - --cohort-file cohort_FD.csv \ - --relative-root /home/username/myProject/data \ - --analysis-name mylm \ - --input-hdf5 FD.h5 \ - --output-dir FD_stats -``` -Now you can view the results in folder "FD_stats" in `mrview`. - -> ⚠️ ⚠️ WARNING ⚠️ ⚠️ : If there are existing images in the `--output-dir`, those existing images won't be overwritten! See [notes regarding "Existing output folder and output images"](#existing-output-folder-and-output-images) for more. - -### For additional description: -You can refer to `--help` for additional information: -``` console -confixel --help -fixelstats_write --help -``` - - -## Other notes -### Export from `.h5` to `.mif` (`fixelstats_write`) -#### Existing output folder and output images -⚠️ ⚠️ WARNING ⚠️ ⚠️ -* If there are existing images in the output folder and they have the same filenames as those to be saved, the existing images won't be overwritten. This is because in `modelarrayio.fixels.nifti2_to_mif()`, we did not turn on `-force` in `mrconvert` (i.e., output files won't be overwritten). -* In addition, if the output folder already exists, `fixelstats_write` will not delete it or create a new one. You will only get a message saying "WARNING: Output directory exists". Therefore, any existing files in the output folder will be kept as it is. -* So, if the output folder already exists, you may consider manually deleting it before using `fixelstats_write` to save new images. diff --git a/notebooks/walkthrough_voxel-wise_data.md b/notebooks/walkthrough_voxel-wise_data.md deleted file mode 100644 index daea8da..0000000 --- a/notebooks/walkthrough_voxel-wise_data.md +++ /dev/null @@ -1,125 +0,0 @@ -# Walkthrough for voxel-wise data conversion - -For voxel-wise data, use the **`convoxel`** command from **ModelArrayIO**. In general, the voxel workflow is very similar to the fixel workflow (`confixel`). - -## Prepare data -To convert (a list of) voxel-wise data from NIfTI format to .h5 format, you need to prepare a cohort CSV file that provides basic information for all NIfTI files you want to include. We recommend that, for each scalar (e.g. FA), prepare one .csv file, and thus getting one .h5 file. - -In addition, unlike the fixel pipeline, you also need to provide these image files: -* one group mask: Only voxels within the group mask will be kept during conversion to .h5 file. -* subject-specific masks (i.e., individual masks): This takes the inconsistent boundary of subject-specific images into account. After conversion, for each subject's scalar mage, voxels outside the subject-specific mask will be set to `NaN`. `ModelArray` will then check if each voxel has sufficient number of subjects to get reliable statistics (see argument `num.subj.lthr.abs` and `num.subj.lthr.rel` in Model fitting functions, e.g., [`ModelArray.lm()`](https://pennlinc.github.io/ModelArray/reference/ModelArray.lm.html)). - * If you don't have subject-specific masks, that's fine; you can use group mask instead (see below for how to achieve this in .csv file). - -### Cohort's csv file (for each scalar) -Each row of a cohort .csv is for one NIfTI file you want to include. The file should at least include these columns (Notes: these column names are fixed, i.e. not user-defined): - -* "scalar_name": which tells us what metric is being analyzed, e.g. FA -* "source_file": which tells us which NIfTI file will be used for this subject -* "source_mask_file": which tells us the filename of subject-specific masks. If you don't have subject-specific masks, simply provide filename of group mask here for each subject. - -### Example -#### Example folder structure -``` -/home/username/myProject/data -| -├── cohort_FA.csv -├── group_mask.nii.gz -│ -├── FA -| ├── sub-01_FA.nii.gz -| ├── sub-02_FA.nii.gz -| ├── sub-03_FA.nii.gz -│ ├── ... -│ -├── individual_masks -| ├── sub-01_mask.nii.gz -| ├── sub-02_mask.nii.gz -| ├── sub-03_mask.nii.gz -| ├── ... -└── ... -``` - -#### Corresponding csv file for scalar FA can look like this: -"cohort_FA.csv" for scalar FA: -| ***scalar_name*** | ***source_file*** | ***source_mask_file*** | subject_id | age | sex | -| :----: | :----: | :----: | :----: | :----: | :----: | -| FA | FA/sub-01_FA.nii.gz | individual_masks/sub-01_mask.nii.gz | sub-01 | 10 | F | -| FA | FA/sub-02_FA.nii.gz | individual_masks/sub-02_mask.nii.gz | sub-02 | 20 | M | -| FA | FA/sub-03_FA.nii.gz | individual_masks/sub-03_mask.nii.gz | sub-03 | 15 | F | -| ... | ... | ... | ... | ... | ... | - -Notes: -* Columns that must be included are highlighted in ***bold and italics***; -* The order of columns can be changed. -* Please make sure the consistency (see examples as below). These strings are case sensitive, too! Either upper case or lower case works, just need to be consistent :) - * Folder name e.g., "FA" in column `source_file` in CSV file v.s. the actual folder name on disk; - * File names in columns `source_file` and `source_mask_file` in CSV file v.s. the actual file names on disk; - * Scalar name e.g., "FA" in column `scalar_name` in CSV file v.s. what you will specify when using functions in `ModelArray`; - -For this case, when running `convoxel`, argument `--relative-root` should be `/home/username/myProject/data` - -## Run `convoxel` and `volumestats_write` -### Convert NIfTI files to an HDF5 (.h5) file -Using above described scenario as an example, for FA dataset: -``` console -# first, activate conda environment where ModelArrayIO is installed: `conda activate ` -convoxel \ - --group-mask-file group_mask.nii.gz \ - --cohort-file cohort_FA.csv \ - --relative-root /home/username/myProject/data \ - --output-hdf5 FA.h5 -``` - -Now you should get the HDF5 file "FA.h5" in folder "/home/username/myProject/data". You may use [`ModelArray`](https://pennlinc.github.io/ModelArray/) to perform statistical analysis. - -### Convert result .h5 file back to NIfTI format -After running `ModelArray` and getting statistical results in FA.h5 file (say, the analysis name is called "mylm"), you can use `volumestats_write` to convert results into a list of NIfTI files in a folder specified by you. - -``` console -# first, activate conda environment where ModelArrayIO is installed: `conda activate ` -volumestats_write \ - --group-mask-file group_mask.nii.gz \ - --cohort-file cohort_FA.csv \ - --relative-root /home/username/myProject/data \ - --analysis-name mylm \ - --input-hdf5 FA.h5 \ - --output-dir FA_stats \ - --output-ext .nii.gz # or ".nii" -``` - -Now you should get the results NIfTI images saved in folder "FA_stats". All the converted volume data are saved with data type float32. You can view the images with the image viewer you like. - -> ⚠️ ⚠️ WARNING ⚠️ ⚠️ : See [notes regarding "Existing output folder and output images"](#existing-output-folder-and-output-images). - -### For additional information: -You can refer to `--help` for additional information: -``` console -convoxel --help -volumestats_write --help -``` - -## Other notes -### `volumestats_write`: convert from `.h5` to NIfTI -#### Existing output folder and output images -⚠️ ⚠️ WARNING ⚠️ ⚠️ -* If the output folder already exists, `volumestats_write` will not delete it or create a new one. You will only get a message saying "WARNING: Output directory exists". Therefore, if there were existing files in the output folder, and they are not part of the current list of images to be saved (e.g., results to be saved were changed, but the output folder name was not changed), these files will be kept as it is and won't be deleted. - * However, for existing files which are still part of the current list to be saved, they will be replaced. This differs from the fixel-wise `fixelstats_write` behavior (which does not overwrite existing `.mif` outputs via `mrconvert` without `-force`). -* So to avoid confusion and better for version controls, if the output folder already exists, you might consider manually deleting it before using `volumestats_write` to save new images. - - -### Image of number of observations used -If you requested `nobs` when running model fitting in `ModelArray`, after conversion back to NIfTI files, you'll get an image called `*_model.nobs.nii*` (number of observations used). With the feature of "subject-specific masks", you'll probably see inhomogeneity in this image. - -### Results for voxels without sufficient subjects (because of subject-specific masks): - - -| software | regular voxel | voxel without sufficient subjects | -| :----: | :----: | :----: | -| python nibabel | nobs: e.g. 209.0; p.value: e.g. 0.010000... | nobs or p.value or 1m.p.value: nan | -| MRtrix mrview | nobs: e.g. 209; p.value: e.g. 0.010000| nobs or p.value or 1m.p.value: ?| -| ITK-snap | nobs: e.g. 209; p.value: e.g. 0.0100 | nobs or p.value or 1m.p.value: 0 | - -Notes: -* `regular voxel` means voxels with sufficient subjects -* Here `nobs` is number of observations from ModelArray, which should be integer. Also, expect it is not constant across voxels if different subject-specific masks were provided -* Although ITK-snap does not display NaN as "?", those voxels without sufficient subjects won't pop out when thresholding p.values (e.g. p.value ranging 0-0.05, 1m.p.value ranging 0.95-1), so they won't be "mixed" with other regular voxels. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9b8dd7a..5727360 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ doc = [ "sphinx_rtd_theme", "sphinxcontrib-apidoc", "sphinxcontrib-bibtex", + "sphinx-gallery", ] test = [ # Lower bounds matter for tox `uv_resolution = lowest-direct` (py311-min CI): @@ -66,11 +67,11 @@ Repository = "https://github.com/PennLINC/ModelArrayIO" Documentation = "https://modelarrayio.readthedocs.io" [project.scripts] -confixel = "modelarrayio.cli.fixels_to_h5:main" -convoxel = "modelarrayio.cli.voxels_to_h5:main" +confixel = "modelarrayio.cli.mif_to_h5:main" +convoxel = "modelarrayio.cli.nifti_to_h5:main" concifti = "modelarrayio.cli.cifti_to_h5:main" -fixelstats_write = "modelarrayio.cli.h5_to_fixels:main" -volumestats_write = "modelarrayio.cli.h5_to_voxels:main" +fixelstats_write = "modelarrayio.cli.h5_to_mif:main" +volumestats_write = "modelarrayio.cli.h5_to_nifti:main" ciftistats_write = "modelarrayio.cli.h5_to_cifti:main" # diff --git a/src/modelarrayio/cli/cifti_to_h5.py b/src/modelarrayio/cli/cifti_to_h5.py index e61959e..ff3fc86 100644 --- a/src/modelarrayio/cli/cifti_to_h5.py +++ b/src/modelarrayio/cli/cifti_to_h5.py @@ -1,3 +1,5 @@ +"""Convert CIFTI2 dscalar data to an HDF5 file.""" + import argparse import logging import os @@ -12,7 +14,6 @@ add_cohort_arg, add_output_hdf5_arg, add_output_tiledb_arg, - add_relative_root_arg, add_s3_workers_arg, add_scalar_columns_arg, add_storage_args, @@ -30,12 +31,11 @@ logger = logging.getLogger(__name__) -def write_storage( +def cifti_to_h5( cohort_file, backend='hdf5', output_hdf5='fixeldb.h5', output_tiledb='arraydb.tdb', - relative_root='/', storage_dtype='float32', compression='gzip', compression_level=4, @@ -63,8 +63,6 @@ def write_storage( Path to a new .h5 file to be written output_tiledb : :obj:`str` Path to a new .tdb file to be written - relative_root : :obj:`str` - Root to which all paths are relative storage_dtype : :obj:`str` Floating type to store values compression : :obj:`str` @@ -99,8 +97,7 @@ def write_storage( status : :obj:`int` Status of the operation. 0 if successful, 1 if failed. """ - cohort_path = os.path.join(relative_root, cohort_file) - cohort_df = pd.read_csv(cohort_path) + cohort_df = pd.read_csv(cohort_file) cohort_long = _cohort_to_long_dataframe(cohort_df, scalar_columns=scalar_columns) if cohort_long.empty: raise ValueError('Cohort file does not contain any scalar entries after normalization.') @@ -109,10 +106,9 @@ def write_storage( raise ValueError('Unable to derive scalar sources from cohort file.') if backend == 'hdf5': - scalars, last_brain_names = _load_cohort_cifti(cohort_long, relative_root, s3_workers) + scalars, last_brain_names = _load_cohort_cifti(cohort_long, s3_workers) - output_file = os.path.join(relative_root, output_hdf5) - f = h5py.File(output_file, 'w') + f = h5py.File(output_hdf5, 'w') greyordinate_table, structure_names = brain_names_to_dataframe(last_brain_names) greyordinatesh5 = f.create_dataset( @@ -140,25 +136,23 @@ def write_storage( h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name]) f.close() - return int(not os.path.exists(output_file)) + return int(not os.path.exists(output_hdf5)) else: - base_uri = os.path.join(relative_root, output_tiledb) - os.makedirs(base_uri, exist_ok=True) + os.makedirs(output_tiledb, exist_ok=True) if not scalar_sources: return 0 # Establish a reference brain axis once to ensure consistent ordering across workers. _first_scalar, first_sources = next(iter(scalar_sources.items())) - first_path = os.path.join(relative_root, first_sources[0]) + first_path = first_sources[0] _, reference_brain_names = extract_cifti_scalar_data(first_path) def _process_scalar_job(scalar_name, source_files): dataset_path = f'scalars/{scalar_name}/values' rows = [] for source_file in source_files: - scalar_file = os.path.join(relative_root, source_file) cifti_data, _ = extract_cifti_scalar_data( - scalar_file, reference_brain_names=reference_brain_names + source_file, reference_brain_names=reference_brain_names ) rows.append(cifti_data) @@ -167,7 +161,7 @@ def _process_scalar_job(scalar_name, source_files): return scalar_name num_items = rows[0].shape[0] tiledb_storage.create_empty_scalar_matrix_array( - base_uri, + output_tiledb, dataset_path, num_subjects, num_items, @@ -180,8 +174,8 @@ def _process_scalar_job(scalar_name, source_files): sources_list=source_files, ) # write column names array for ModelArray compatibility - tiledb_storage.write_column_names(base_uri, scalar_name, source_files) - uri = os.path.join(base_uri, dataset_path) + tiledb_storage.write_column_names(output_tiledb, scalar_name, source_files) + uri = os.path.join(output_tiledb, dataset_path) tiledb_storage.write_rows_in_column_stripes(uri, rows) return scalar_name @@ -211,10 +205,12 @@ def _process_scalar_job(scalar_name, source_files): def get_parser(): - parser = argparse.ArgumentParser(description='Create a hdf5 file of CIDTI2 dscalar data') + parser = argparse.ArgumentParser( + description='Create a hdf5 file of CIDTI2 dscalar data', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) add_cohort_arg(parser) add_scalar_columns_arg(parser) - add_relative_root_arg(parser) add_output_hdf5_arg(parser, default_name='fixelarray.h5') add_output_tiledb_arg(parser, default_name='arraydb.tdb') add_backend_arg(parser) @@ -243,4 +239,4 @@ def main(): level=getattr(logging, str(log_level).upper(), logging.INFO), format='[%(levelname)s] %(name)s: %(message)s', ) - return write_storage(**kwargs) + return cifti_to_h5(**kwargs) diff --git a/src/modelarrayio/cli/h5_to_cifti.py b/src/modelarrayio/cli/h5_to_cifti.py index da2d536..f5eb7d5 100644 --- a/src/modelarrayio/cli/h5_to_cifti.py +++ b/src/modelarrayio/cli/h5_to_cifti.py @@ -1,18 +1,23 @@ +"""Convert HDF5 file to CIFTI2 dscalar data.""" + import argparse import logging import os +from functools import partial import h5py import nibabel as nb import pandas as pd +from modelarrayio.cli.parser_utils import _is_file + logger = logging.getLogger(__name__) -def _h5_to_ciftis(example_cifti, h5_file, analysis_name, cifti_output_dir): +def h5_to_cifti(example_cifti, in_file, analysis_name, output_dir): """Write the contents of an hdf5 file to a fixels directory. - The ``h5_file`` parameter should point to an HDF5 file that contains at least two + The ``in_file`` parameter should point to an HDF5 file that contains at least two datasets. There must be one called ``results/results_matrix``, that contains a matrix of fixel results. Each column contains a single result and each row is a fixel. This matrix should be of type float. The second required dataset must be @@ -28,7 +33,7 @@ def _h5_to_ciftis(example_cifti, h5_file, analysis_name, cifti_output_dir): ========== example_cifti: pathlike abspath to a scalar cifti file. Its header is used as a template - h5_file: str + in_file: str abspath to an h5 file that contains statistical results and their metadata. analysis_name: str the name for the analysis results to be saved @@ -41,12 +46,9 @@ def _h5_to_ciftis(example_cifti, h5_file, analysis_name, cifti_output_dir): """ # Get a template nifti image. cifti = nb.load(example_cifti) - h5_data = h5py.File(h5_file, 'r') + h5_data = h5py.File(in_file, 'r') results_matrix = h5_data['results/' + analysis_name + '/results_matrix'] names_data = results_matrix.attrs['colnames'] # NOTE: results_matrix: need to be transposed... - # print(results_matrix.shape) - - # print(h5_data['results/' + analysis_name + '/results_matrix'].attrs['column_names']) try: results_names = names_data.tolist() @@ -55,13 +57,14 @@ def _h5_to_ciftis(example_cifti, h5_file, analysis_name, cifti_output_dir): results_names = [f'component{n + 1:03d}' for n in range(results_matrix.shape[0])] # Make output directory if it does not exist - if not os.path.isdir(cifti_output_dir): - os.mkdir(cifti_output_dir) + if not os.path.isdir(output_dir): + os.mkdir(output_dir) for result_col, result_name in enumerate(results_names): valid_result_name = result_name.replace(' ', '_').replace('/', '_') out_cifti = os.path.join( - cifti_output_dir, analysis_name + '_' + valid_result_name + '.dscalar.nii' + output_dir, + f'{analysis_name}_{valid_result_name}.dscalar.nii', ) temp_cifti2 = nb.Cifti2Image( results_matrix[result_col, :].reshape(1, -1), @@ -71,12 +74,12 @@ def _h5_to_ciftis(example_cifti, h5_file, analysis_name, cifti_output_dir): temp_cifti2.to_filename(out_cifti) # if this result is p.value, also write out 1-p.value (1m.p.value) - if ( - 'p.value' in valid_result_name - ): # the result name contains "p.value" (from R package broom) + # the result name contains "p.value" (from R package broom) + if 'p.value' in valid_result_name: valid_result_name_1mpvalue = valid_result_name.replace('p.value', '1m.p.value') out_cifti_1mpvalue = os.path.join( - cifti_output_dir, analysis_name + '_' + valid_result_name_1mpvalue + '.dscalar.nii' + output_dir, + f'{analysis_name}_{valid_result_name_1mpvalue}.dscalar.nii', ) output_mifvalues_1mpvalue = 1 - results_matrix[result_col, :] # 1 minus temp_nifti2_1mpvalue = nb.Cifti2Image( @@ -92,47 +95,34 @@ def main(): parser = get_parser() args = parser.parse_args() - out_cifti_dir = os.path.abspath(args.output_dir) # absolute path for output dir - - if os.path.exists(out_cifti_dir): + if os.path.exists(args.output_dir): print('WARNING: Output directory exists') - os.makedirs(out_cifti_dir, exist_ok=True) + os.makedirs(args.output_dir, exist_ok=True) # Get an example cifti - if args.example_cifti is None: + example_cifti = args.example_cifti + if example_cifti is None: logger.warning( 'No example cifti file provided, using the first cifti file from the cohort file' ) cohort_df = pd.read_csv(args.cohort_file) - example_cifti = os.path.join(args.relative_root, cohort_df['source_file'][0]) - else: - example_cifti = args.example_cifti - if not os.path.exists(example_cifti): - raise ValueError(f'Example cifti file {example_cifti} does not exist') + example_cifti = cohort_df['source_file'][0] - h5_input = args.input_hdf5 - analysis_name = args.analysis_name - _h5_to_ciftis(example_cifti, h5_input, analysis_name, out_cifti_dir) + h5_to_cifti( + example_cifti=example_cifti, + in_file=args.in_file, + analysis_name=args.analysis_name, + output_dir=args.output_dir, + ) def get_parser(): parser = argparse.ArgumentParser( - description='Create a directory with cifti results from an hdf5 file' - ) - parser.add_argument( - '--cohort-file', - '--cohort_file', - help='Path to a csv with demographic info and paths to data.', - ) - parser.add_argument( - '--relative-root', - '--relative_root', - help=( - 'Root to which all paths are relative, i.e. defining the (absolute) path to root ' - 'directory of index_file, directions_file, cohort_file, input_hdf5, and output_dir.' - ), - type=os.path.abspath, + description='Create a directory with cifti results from an hdf5 file', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) + IsFile = partial(_is_file, parser=parser) + parser.add_argument( '--analysis-name', '--analysis_name', @@ -142,19 +132,37 @@ def get_parser(): '--input-hdf5', '--input_hdf5', help='Name of HDF5 (.h5) file where results outputs are saved.', + type=IsFile, + dest='in_file', ) parser.add_argument( '--output-dir', '--output_dir', help=( - 'Fixel directory where outputs will be saved. ' + 'Directory where outputs will be saved. ' 'If the directory does not exist, it will be automatically created.' ), ) - parser.add_argument( + + example_cifti_group = parser.add_mutually_exclusive_group() + example_cifti_group.add_argument( + '--cohort-file', + '--cohort_file', + help=( + 'Path to a csv with demographic info and paths to data. ' + 'Used to select an example CIFTI file if no example CIFTI file is provided.' + ), + type=IsFile, + required=False, + default=None, + ) + example_cifti_group.add_argument( '--example-cifti', '--example_cifti', help='Path to an example cifti file.', required=False, + type=IsFile, + default=None, ) + return parser diff --git a/src/modelarrayio/cli/h5_to_fixels.py b/src/modelarrayio/cli/h5_to_mif.py similarity index 70% rename from src/modelarrayio/cli/h5_to_fixels.py rename to src/modelarrayio/cli/h5_to_mif.py index 48e4e6a..f5a4b21 100644 --- a/src/modelarrayio/cli/h5_to_fixels.py +++ b/src/modelarrayio/cli/h5_to_mif.py @@ -1,18 +1,25 @@ +"""Convert HDF5 file to MIF data.""" + import argparse +import logging import os import shutil +from functools import partial import h5py import nibabel as nb import pandas as pd +from modelarrayio.cli.parser_utils import _is_file from modelarrayio.utils.fixels import mif_to_nifti2, nifti2_to_mif +logger = logging.getLogger(__name__) + -def h5_to_mifs(example_mif, h5_file, analysis_name, fixel_output_dir): +def h5_to_mif(example_mif, in_file, analysis_name, output_dir): """Writes the contents of an hdf5 file to a fixels directory. - The ``h5_file`` parameter should point to an HDF5 file that contains at least two + The ``in_file`` parameter should point to an HDF5 file that contains at least two datasets. There must be one called ``results/results_matrix``, that contains a matrix of fixel results. Each column contains a single result and each row is a fixel. This matrix should be of type float. The second required dataset must be @@ -28,11 +35,11 @@ def h5_to_mifs(example_mif, h5_file, analysis_name, fixel_output_dir): ========== example_mif: str abspath to a scalar mif file. Its header is used as a template - h5_file: str + in_file: str abspath to an h5 file that contains statistical results and their metadata. analysis_name: str the name for the analysis results to be saved - fixel_output_dir: str + output_dir: str abspath to where the output fixel data will go. the index and directions mif files should already be copied here. @@ -42,7 +49,7 @@ def h5_to_mifs(example_mif, h5_file, analysis_name, fixel_output_dir): """ # Get a template nifti image. nifti2_img, _ = mif_to_nifti2(example_mif) - h5_data = h5py.File(h5_file, 'r') + h5_data = h5py.File(in_file, 'r') results_matrix = h5_data['results/' + analysis_name + '/results_matrix'] names_data = results_matrix.attrs['colnames'] # NOTE: results_matrix: need to be transposed... # print(results_matrix.shape) @@ -56,12 +63,12 @@ def h5_to_mifs(example_mif, h5_file, analysis_name, fixel_output_dir): results_names = [f'component{n + 1:03d}' for n in range(results_matrix.shape[0])] # Make output directory if it does not exist - if not os.path.isdir(fixel_output_dir): - os.mkdir(fixel_output_dir) + if not os.path.isdir(output_dir): + os.mkdir(output_dir) for result_col, result_name in enumerate(results_names): valid_result_name = result_name.replace(' ', '_').replace('/', '_') - out_mif = os.path.join(fixel_output_dir, analysis_name + '_' + valid_result_name + '.mif') + out_mif = os.path.join(output_dir, f'{analysis_name}_{valid_result_name}.mif') temp_nifti2 = nb.Nifti2Image( results_matrix[result_col, :].reshape(-1, 1, 1), nifti2_img.affine, @@ -70,12 +77,11 @@ def h5_to_mifs(example_mif, h5_file, analysis_name, fixel_output_dir): nifti2_to_mif(temp_nifti2, out_mif) # if this result is p.value, also write out 1-p.value (1m.p.value) - if ( - 'p.value' in valid_result_name - ): # the result name contains "p.value" (from R package broom) + # the result name contains "p.value" (from R package broom) + if 'p.value' in valid_result_name: valid_result_name_1mpvalue = valid_result_name.replace('p.value', '1m.p.value') out_mif_1mpvalue = os.path.join( - fixel_output_dir, analysis_name + '_' + valid_result_name_1mpvalue + '.mif' + output_dir, f'{analysis_name}_{valid_result_name_1mpvalue}.mif' ) output_mifvalues_1mpvalue = 1 - results_matrix[result_col, :] # 1 minus temp_nifti2_1mpvalue = nb.Nifti2Image( @@ -90,59 +96,58 @@ def main(): parser = get_parser() args = parser.parse_args() - # absolute path for output dir - out_fixel_dir = os.path.join(args.relative_root, args.output_dir) - - if os.path.exists(out_fixel_dir): + if os.path.exists(args.output_dir): print('WARNING: Output directory exists') - os.makedirs(out_fixel_dir, exist_ok=True) + os.makedirs(args.output_dir, exist_ok=True) # Copy in the index and directions shutil.copyfile( - os.path.join(args.relative_root, args.directions_file), - os.path.join(out_fixel_dir, os.path.split(args.directions_file)[1]), + args.directions_file, + os.path.join(args.output_dir, os.path.basename(args.directions_file)), ) shutil.copyfile( - os.path.join(args.relative_root, args.index_file), - os.path.join(out_fixel_dir, os.path.split(args.index_file)[1]), + args.index_file, + os.path.join(args.output_dir, os.path.basename(args.index_file)), ) # Get an example mif file - cohort_df = pd.read_csv(os.path.join(args.relative_root, args.cohort_file)) - example_mif = os.path.join(args.relative_root, cohort_df['source_file'][0]) - h5_input = os.path.join(args.relative_root, args.input_hdf5) - analysis_name = args.analysis_name - h5_to_mifs(example_mif, h5_input, analysis_name, out_fixel_dir) + cohort_df = pd.read_csv(args.cohort_file) + example_mif = cohort_df['source_file'][0] + h5_to_mif( + example_mif=example_mif, + in_file=args.in_file, + analysis_name=args.analysis_name, + output_dir=args.output_dir, + ) def get_parser(): - parser = argparse.ArgumentParser(description='Create a fixel directory from an hdf5 file') + parser = argparse.ArgumentParser( + description='Create a fixel directory from an hdf5 file', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + IsFile = partial(_is_file, parser=parser) + parser.add_argument( '--index-file', '--index_file', help='Index File', required=True, + type=IsFile, ) parser.add_argument( '--directions-file', '--directions_file', help='Directions File', required=True, + type=IsFile, ) parser.add_argument( '--cohort-file', '--cohort_file', help='Path to a csv with demographic info and paths to data.', required=True, - ) - parser.add_argument( - '--relative-root', - '--relative_root', - help=( - 'Root to which all paths are relative, i.e. defining the (absolute) path to root ' - 'directory of index_file, directions_file, cohort_file, input_hdf5, and output_dir.' - ), - type=os.path.abspath, + type=IsFile, ) parser.add_argument( '--analysis-name', @@ -153,6 +158,8 @@ def get_parser(): '--input-hdf5', '--input_hdf5', help='Name of HDF5 (.h5) file where results outputs are saved.', + type=IsFile, + dest='in_file', ) parser.add_argument( '--output-dir', diff --git a/src/modelarrayio/cli/h5_to_voxels.py b/src/modelarrayio/cli/h5_to_nifti.py similarity index 84% rename from src/modelarrayio/cli/h5_to_voxels.py rename to src/modelarrayio/cli/h5_to_nifti.py index d1ac19d..0614471 100644 --- a/src/modelarrayio/cli/h5_to_voxels.py +++ b/src/modelarrayio/cli/h5_to_nifti.py @@ -1,17 +1,20 @@ +"""Convert HDF5 file to NIfTI data.""" + import argparse import logging import os +from functools import partial import h5py import nibabel as nb import numpy as np -from modelarrayio.cli.parser_utils import add_relative_root_arg +from modelarrayio.cli.parser_utils import _is_file logger = logging.getLogger(__name__) -def h5_to_volumes(h5_file, analysis_name, group_mask_file, output_extension, volume_output_dir): +def h5_to_nifti(in_file, analysis_name, group_mask_file, output_extension, output_dir): """Convert stat results in .h5 file to a list of volume (.nii or .nii.gz) files.""" data_type_tosave = np.float32 @@ -27,7 +30,7 @@ def h5_to_volumes(h5_file, analysis_name, group_mask_file, output_extension, vol ) # modify the data type (mask's data type could be uint8...) # results in .h5 file: - h5_data = h5py.File(h5_file, 'r') + h5_data = h5py.File(in_file, 'r') results_matrix = h5_data['results/' + analysis_name + '/results_matrix'] # NOTE: results_matrix may need to be transposed depending on writer conventions @@ -84,15 +87,15 @@ def _decode_names(arr): results_names = [f'component{n + 1:03d}' for n in range(results_matrix.shape[0])] # # Make output directory if it does not exist # has been done in h5_to_volumes_wrapper() - # if os.path.isdir(volume_output_dir) == False: - # os.mkdir(volume_output_dir) + # if os.path.isdir(output_dir) == False: + # os.mkdir(output_dir) # for loop: save stat metric results one by one: for result_col, result_name in enumerate(results_names): valid_result_name = result_name.replace(' ', '_').replace('/', '_') out_file = os.path.join( - volume_output_dir, analysis_name + '_' + valid_result_name + output_extension + output_dir, analysis_name + '_' + valid_result_name + output_extension ) output = np.zeros(group_mask_matrix.shape) data_tosave = results_matrix[result_col, :] @@ -108,7 +111,7 @@ def _decode_names(arr): if 'p.value' in valid_result_name: valid_result_name_1mpvalue = valid_result_name.replace('p.value', '1m.p.value') out_file_1mpvalue = os.path.join( - volume_output_dir, + output_dir, analysis_name + '_' + valid_result_name_1mpvalue + output_extension, ) output_1mpvalue = np.zeros(group_mask_matrix.shape) @@ -127,43 +130,33 @@ def main(): parser = get_parser() args = parser.parse_args() - volume_output_dir = os.path.join( - args.relative_root, args.output_dir - ) # absolute path for output dir - - if os.path.exists(volume_output_dir): + if os.path.exists(args.output_dir): print('WARNING: Output directory exists') - os.makedirs(volume_output_dir, exist_ok=True) - - # any files to copy? - - # other arguments: - group_mask_file = os.path.join(args.relative_root, args.group_mask_file) - h5_input = os.path.join(args.relative_root, args.input_hdf5) - analysis_name = args.analysis_name - output_extension = args.output_ext + os.makedirs(args.output_dir, exist_ok=True) - # call function: - h5_to_volumes(h5_input, analysis_name, group_mask_file, output_extension, volume_output_dir) + h5_to_nifti(*vars(args)) def get_parser(): parser = argparse.ArgumentParser( - description='Convert statistical results from an hdf5 file to a volume data (NIfTI file)' + description='Convert statistical results from an hdf5 file to a volume data (NIfTI file)', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) + IsFile = partial(_is_file, parser=parser) parser.add_argument( '--group-mask-file', '--group_mask_file', help='Path to a group mask file', required=True, + type=IsFile, ) parser.add_argument( '--cohort-file', '--cohort_file', help='Path to a csv with demographic info and paths to data.', required=True, + type=IsFile, ) - add_relative_root_arg(parser) parser.add_argument( '--analysis-name', '--analysis_name', @@ -173,6 +166,8 @@ def get_parser(): '--input-hdf5', '--input_hdf5', help='Name of HDF5 (.h5) file where results outputs are saved.', + type=IsFile, + dest='in_file', ) parser.add_argument( '--output-dir', @@ -189,6 +184,7 @@ def get_parser(): 'The extension for output volume data. ' 'Options are .nii.gz (default) and .nii. Please provide the prefix dot.' ), + choices=['.nii.gz', '.nii'], default='.nii.gz', ) return parser diff --git a/src/modelarrayio/cli/fixels_to_h5.py b/src/modelarrayio/cli/mif_to_h5.py similarity index 85% rename from src/modelarrayio/cli/fixels_to_h5.py rename to src/modelarrayio/cli/mif_to_h5.py index ff99d3d..d05bb92 100644 --- a/src/modelarrayio/cli/fixels_to_h5.py +++ b/src/modelarrayio/cli/mif_to_h5.py @@ -1,24 +1,29 @@ +"""Convert MIF data to an HDF5 file.""" + import argparse import logging -import os from collections import defaultdict +from functools import partial +from pathlib import Path import h5py import pandas as pd from tqdm import tqdm +from modelarrayio.cli.parser_utils import _is_file from modelarrayio.storage import h5_storage, tiledb_storage from modelarrayio.utils.fixels import gather_fixels, mif_to_nifti2 +logger = logging.getLogger(__name__) + -def write_storage( +def mif_to_h5( index_file, directions_file, cohort_file, backend='hdf5', - output_hdf5='fixeldb.h5', - output_tiledb='arraydb.tdb', - relative_root='/', + output_hdf5=Path('fixeldb.h5'), + output_tiledb=Path('arraydb.tdb'), storage_dtype='float32', compression='gzip', compression_level=4, @@ -35,20 +40,18 @@ def write_storage( Parameters ---------- - index_file : :obj:`str` + index_file : :obj:`pathlib.Path` Path to a Nifti2 index file - directions_file : :obj:`str` + directions_file : :obj:`pathlib.Path` Path to a Nifti2 directions file - cohort_file : :obj:`str` + cohort_file : :obj:`pathlib.Path` Path to a csv with demographic info and paths to data backend : :obj:`str` Backend to use for storage - output_hdf5 : :obj:`str` + output_hdf5 : :obj:`pathlib.Path` Path to a new .h5 file to be written - output_tiledb : :obj:`str` + output_tiledb : :obj:`pathlib.Path` Path to a new .tdb file to be written - relative_root : :obj:`str` - Root to which all paths are relative storage_dtype : :obj:`str` Floating type to store values compression : :obj:`str` @@ -78,12 +81,10 @@ def write_storage( Status of the operation. 0 if successful, 1 if failed. """ # gather fixel data - fixel_table, voxel_table = gather_fixels( - os.path.join(relative_root, index_file), os.path.join(relative_root, directions_file) - ) + fixel_table, voxel_table = gather_fixels(index_file, directions_file) # gather cohort data - cohort_df = pd.read_csv(os.path.join(relative_root, cohort_file)) + cohort_df = pd.read_csv(cohort_file) # upload each cohort's data scalars = defaultdict(list) @@ -91,7 +92,7 @@ def write_storage( print('Extracting .mif data...') # ix: index of row (start from 0); row: one row of data for _ix, row in tqdm(cohort_df.iterrows(), total=cohort_df.shape[0]): - scalar_file = os.path.join(relative_root, row['source_file']) + scalar_file = row['source_file'] _scalar_img, scalar_data = mif_to_nifti2(scalar_file) scalars[row['scalar_name']].append(scalar_data) # append to specific scalar_name # append source mif filename to specific scalar_name @@ -99,8 +100,7 @@ def write_storage( # Write the output if backend == 'hdf5': - output_file = os.path.join(relative_root, output_hdf5) - f = h5py.File(output_file, 'w') + f = h5py.File(output_hdf5, 'w') fixelsh5 = f.create_dataset(name='fixels', data=fixel_table.to_numpy().T) fixelsh5.attrs['column_names'] = list(fixel_table.columns) @@ -127,17 +127,16 @@ def write_storage( h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name]) f.close() - return int(not os.path.exists(output_file)) + return int(not output_hdf5.exists()) else: - base_uri = os.path.join(relative_root, output_tiledb) - os.makedirs(base_uri, exist_ok=True) + output_tiledb.mkdir(parents=True, exist_ok=True) for scalar_name in scalars.keys(): num_subjects = len(scalars[scalar_name]) num_items = scalars[scalar_name][0].shape[0] if num_subjects > 0 else 0 dataset_path = f'scalars/{scalar_name}/values' tiledb_storage.create_empty_scalar_matrix_array( - base_uri, + output_tiledb, dataset_path, num_subjects, num_items, @@ -149,41 +148,39 @@ def write_storage( target_tile_mb=tdb_target_tile_mb, sources_list=sources_lists[scalar_name], ) - uri = os.path.join(base_uri, dataset_path) + uri = output_tiledb / dataset_path tiledb_storage.write_rows_in_column_stripes(uri, scalars[scalar_name]) return 0 def get_parser(): - parser = argparse.ArgumentParser(description='Create a hdf5 file of fixel data') + parser = argparse.ArgumentParser( + description='Create a hdf5 file of fixel data', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + IsFile = partial(_is_file, parser=parser) + parser.add_argument( '--index-file', '--index_file', help='Index File', required=True, + type=IsFile, ) parser.add_argument( '--directions-file', '--directions_file', help='Directions File', required=True, + type=IsFile, ) parser.add_argument( '--cohort-file', '--cohort_file', help='Path to a csv with demographic info and paths to data.', required=True, - ) - parser.add_argument( - '--relative-root', - '--relative_root', - help=( - 'Root to which all paths are relative, i.e. defining the (absolute) ' - 'path to root directory of index_file, directions_file, cohort_file, and output_hdf5.' - ), - type=os.path.abspath, - default='/inputs/', + type=IsFile, ) parser.add_argument( '--output-hdf5', @@ -307,4 +304,4 @@ def main(): level=getattr(logging, str(log_level).upper(), logging.INFO), format='[%(levelname)s] %(name)s: %(message)s', ) - return write_storage(**kwargs) + return mif_to_h5(**kwargs) diff --git a/src/modelarrayio/cli/voxels_to_h5.py b/src/modelarrayio/cli/nifti_to_h5.py similarity index 85% rename from src/modelarrayio/cli/voxels_to_h5.py rename to src/modelarrayio/cli/nifti_to_h5.py index a68804b..29b6ed1 100644 --- a/src/modelarrayio/cli/voxels_to_h5.py +++ b/src/modelarrayio/cli/nifti_to_h5.py @@ -1,6 +1,9 @@ +"""Convert NIfTI data to an HDF5 file.""" + import argparse import logging import os +from functools import partial import h5py import nibabel as nb @@ -8,11 +11,11 @@ import pandas as pd from modelarrayio.cli.parser_utils import ( + _is_file, add_backend_arg, add_cohort_arg, add_output_hdf5_arg, add_output_tiledb_arg, - add_relative_root_arg, add_s3_workers_arg, add_storage_args, add_tiledb_storage_args, @@ -23,13 +26,12 @@ logger = logging.getLogger(__name__) -def write_storage( +def nifti_to_h5( group_mask_file, cohort_file, backend='hdf5', output_hdf5='voxeldb.h5', output_tiledb='arraydb.tdb', - relative_root='/', storage_dtype='float32', compression='gzip', compression_level=4, @@ -53,8 +55,6 @@ def write_storage( Path to a CSV with demographic info and paths to data. output_hdf5: str Path to a new .h5 file to be written. - relative_root: str - Path to which group_mask_file and cohort_file (and its contents) are relative. storage_dtype: str Floating type to store values. Options: 'float32' (default), 'float64'. compression: str @@ -70,10 +70,10 @@ def write_storage( Target chunk size in MiB when auto-computing chunk_voxels. Default 2.0. """ # gather cohort data - cohort_df = pd.read_csv(os.path.join(relative_root, cohort_file)) + cohort_df = pd.read_csv(cohort_file) # Load the group mask image to define the rows of the matrix - group_mask_img = nb.load(os.path.join(relative_root, group_mask_file)) + group_mask_img = nb.load(group_mask_file) # get_fdata(): get matrix data in float format group_mask_matrix = group_mask_img.get_fdata() > 0 # np.nonzero() returns the coords of nonzero elements; @@ -93,17 +93,14 @@ def write_storage( # upload each cohort's data print('Extracting NIfTI data...') - scalars, sources_lists = _load_cohort_voxels( - cohort_df, group_mask_matrix, relative_root, s3_workers - ) + scalars, sources_lists = _load_cohort_voxels(cohort_df, group_mask_matrix, s3_workers) # Write the output: if backend == 'hdf5': - output_file = os.path.join(relative_root, output_hdf5) - output_dir = os.path.dirname(output_file) + output_dir = os.path.dirname(output_hdf5) if not os.path.exists(output_dir): os.makedirs(output_dir, exist_ok=True) - f = h5py.File(output_file, 'w') + f = h5py.File(output_hdf5, 'w') voxelsh5 = f.create_dataset(name='voxels', data=voxel_table.to_numpy().T) voxelsh5.attrs['column_names'] = list(voxel_table.columns) @@ -127,11 +124,10 @@ def write_storage( h5_storage.write_rows_in_column_stripes(dset, scalars[scalar_name]) f.close() - return int(not os.path.exists(output_file)) + return int(not os.path.exists(output_hdf5)) else: # TileDB backend - base_uri = os.path.join(relative_root, output_tiledb) - os.makedirs(base_uri, exist_ok=True) + os.makedirs(output_tiledb, exist_ok=True) # Store voxel coordinates as a small TileDB array (optional): # we store as metadata on base group @@ -143,7 +139,7 @@ def write_storage( num_voxels = scalars[scalar_name][0].shape[0] if num_subjects > 0 else 0 dataset_path = f'scalars/{scalar_name}/values' tiledb_storage.create_empty_scalar_matrix_array( - base_uri, + output_tiledb, dataset_path, num_subjects, num_voxels, @@ -156,21 +152,25 @@ def write_storage( sources_list=sources_lists[scalar_name], ) # Stripe-write - uri = os.path.join(base_uri, dataset_path) + uri = os.path.join(output_tiledb, dataset_path) tiledb_storage.write_rows_in_column_stripes(uri, scalars[scalar_name]) return 0 def get_parser(): - parser = argparse.ArgumentParser(description='Create a hdf5 file of volume data') + parser = argparse.ArgumentParser( + description='Create a hdf5 file of volume data', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + IsFile = partial(_is_file, parser=parser) parser.add_argument( '--group-mask-file', '--group_mask_file', help='Path to a group mask file', required=True, + type=IsFile, ) add_cohort_arg(parser) - add_relative_root_arg(parser) add_output_hdf5_arg(parser, default_name='fixelarray.h5') add_output_tiledb_arg(parser, default_name='arraydb.tdb') add_backend_arg(parser) @@ -189,4 +189,4 @@ def main(): level=getattr(logging, str(log_level).upper(), logging.INFO), format='[%(levelname)s] %(name)s: %(message)s', ) - return write_storage(**kwargs) + return nifti_to_h5(**kwargs) diff --git a/src/modelarrayio/cli/parser_utils.py b/src/modelarrayio/cli/parser_utils.py index d5bd007..2ae0944 100644 --- a/src/modelarrayio/cli/parser_utils.py +++ b/src/modelarrayio/cli/parser_utils.py @@ -1,18 +1,5 @@ -import os.path as op - - -def add_relative_root_arg(parser): - parser.add_argument( - '--relative-root', - '--relative_root', - help=( - 'Root to which all paths are relative, i.e. defining the (absolute) path to ' - 'root directory of inputs and outputs.' - ), - type=op.abspath, - default='/inputs/', - ) - return parser +from functools import partial +from pathlib import Path def add_output_hdf5_arg(parser, default_name='fixelarray.h5'): @@ -31,6 +18,7 @@ def add_cohort_arg(parser): '--cohort_file', help='Path to a csv with demographic info and paths to data.', required=True, + type=partial(_is_file, parser=parser), ) return parser @@ -63,7 +51,9 @@ def add_storage_args(parser): help='Disable HDF5 shuffle filter (enabled by default if compression is used).', default=True, ) - parser.add_argument( + + chunk_allocation_group = parser.add_mutually_exclusive_group() + chunk_allocation_group.add_argument( '--chunk-voxels', '--chunk_voxels', type=int, @@ -73,13 +63,14 @@ def add_storage_args(parser): ), default=0, ) - parser.add_argument( + chunk_allocation_group.add_argument( '--target-chunk-mb', '--target_chunk_mb', type=float, help='Target chunk size in MiB when auto-computing item chunk length. Default 2.0', default=2.0, ) + parser.add_argument( '--log-level', '--log_level', @@ -105,10 +96,7 @@ def add_output_tiledb_arg(parser, default_name='arraydb.tdb'): parser.add_argument( '--output-tiledb', '--output_tiledb', - help=( - 'Base URI (directory) where TileDB arrays will be created. ' - 'If relative, it is joined to --relative-root.' - ), + help='Directory where TileDB arrays will be created.', default=default_name, ) return parser @@ -136,7 +124,9 @@ def add_tiledb_storage_args(parser): help='Disable TileDB shuffle filter (enabled by default).', default=True, ) - parser.add_argument( + + tile_allocation_group = parser.add_mutually_exclusive_group() + tile_allocation_group.add_argument( '--tdb-tile-voxels', '--tdb_tile_voxels', type=int, @@ -146,7 +136,7 @@ def add_tiledb_storage_args(parser): ), default=0, ) - parser.add_argument( + tile_allocation_group.add_argument( '--tdb-target-tile-mb', '--tdb_target_tile_mb', type=float, @@ -182,3 +172,18 @@ def add_scalar_columns_arg(parser): ), ) return parser + + +def _path_exists(path, parser): + """Ensure a given path exists.""" + if path is None or not Path(path).exists(): + raise parser.error(f'Path does not exist: <{path}>.') + return Path(path).absolute() + + +def _is_file(path, parser): + """Ensure a given path exists and it is a file.""" + path = _path_exists(path, parser) + if not path.is_file(): + raise parser.error(f'Path should point to a file (or symlink of file): <{path}>.') + return path diff --git a/src/modelarrayio/storage/h5_storage.py b/src/modelarrayio/storage/h5_storage.py index 1f540be..5b721bb 100644 --- a/src/modelarrayio/storage/h5_storage.py +++ b/src/modelarrayio/storage/h5_storage.py @@ -207,7 +207,9 @@ def write_rows_in_column_stripes(dset, rows): with logging_redirect_tqdm(): for start in tqdm( range(0, num_elements, stripe_width), - bar_format='{percentage:3.0f}% {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]', + bar_format=( + '{percentage:3.0f}% {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]' + ), ascii=True, mininterval=max(1, (num_elements / stripe_width) // 200), ): diff --git a/src/modelarrayio/utils/cifti.py b/src/modelarrayio/utils/cifti.py index ef1d5a0..0501735 100644 --- a/src/modelarrayio/utils/cifti.py +++ b/src/modelarrayio/utils/cifti.py @@ -1,6 +1,5 @@ """Utility functions for CIFTI data.""" -import os.path as op from collections import OrderedDict, defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed @@ -9,7 +8,7 @@ import pandas as pd from tqdm import tqdm -from modelarrayio.utils.s3_utils import is_s3_path, load_nibabel +from modelarrayio.utils.s3_utils import load_nibabel def _cohort_to_long_dataframe(cohort_df, scalar_columns=None): @@ -34,7 +33,8 @@ def _cohort_to_long_dataframe(cohort_df, scalar_columns=None): missing = required - set(cohort_df.columns) if missing: raise ValueError( - f'Cohort file must contain columns {sorted(required)} when --scalar-columns is not used.' + f'Cohort file must contain columns {sorted(required)} when ' + '--scalar-columns is not used.' ) long_df = cohort_df[list(required)].copy() @@ -131,7 +131,7 @@ def brain_names_to_dataframe(brain_names): return greyordinate_df, structure_name_strings -def _load_cohort_cifti(cohort_long, relative_root, s3_workers): +def _load_cohort_cifti(cohort_long, s3_workers): """Load all CIFTI scalar rows from the cohort, optionally in parallel. The first file is always loaded serially to obtain the reference brain @@ -144,8 +144,6 @@ def _load_cohort_cifti(cohort_long, relative_root, s3_workers): ---------- cohort_long : :obj:`pandas.DataFrame` Long-format cohort dataframe - relative_root : :obj:`str` - Root to which all paths are relative s3_workers : :obj:`int` Number of workers to use for parallel loading @@ -166,9 +164,8 @@ def _load_cohort_cifti(cohort_long, relative_root, s3_workers): # Load the first file serially to get the reference brain axis first_sn, _, first_src = rows_with_idx[0] - first_path = first_src if is_s3_path(first_src) else op.join(relative_root, first_src) first_data, reference_brain_names = extract_cifti_scalar_data( - load_nibabel(first_path, cifti=True) + load_nibabel(first_src, cifti=True) ) def _worker(job): @@ -180,10 +177,7 @@ def _worker(job): if s3_workers > 1 and len(rows_with_idx) > 1: results = {first_sn: {0: first_data}} - jobs = [ - (sn, subj_idx, src if is_s3_path(src) else op.join(relative_root, src)) - for sn, subj_idx, src in rows_with_idx[1:] - ] + jobs = [(sn, subj_idx, src) for sn, subj_idx, src in rows_with_idx[1:]] with ThreadPoolExecutor(max_workers=s3_workers) as pool: futures = {pool.submit(_worker, job): job for job in jobs} for future in tqdm( @@ -199,10 +193,7 @@ def _worker(job): else: scalars = defaultdict(list) scalars[first_sn].append(first_data) - remaining = [ - (sn, subj_idx, src if is_s3_path(src) else op.join(relative_root, src)) - for sn, subj_idx, src in rows_with_idx[1:] - ] + remaining = [(sn, subj_idx, src) for sn, subj_idx, src in rows_with_idx[1:]] for job in tqdm(remaining, desc='Loading CIFTI data'): sn, subj_idx, arr = _worker(job) scalars[sn].append(arr) diff --git a/src/modelarrayio/utils/fixels.py b/src/modelarrayio/utils/fixels.py index dabe658..513f6f4 100644 --- a/src/modelarrayio/utils/fixels.py +++ b/src/modelarrayio/utils/fixels.py @@ -113,35 +113,33 @@ def gather_fixels(index_file, directions_file): DataFrame with voxel_id, i, j, k """ _index_img, index_data = mif_to_nifti2(index_file) - count_vol = index_data[..., 0].astype( - np.uint32 - ) # number of fixels in each voxel; by index.mif definition - id_vol = index_data[ - ..., 1 - ] # index of the first fixel in this voxel, in the list of all fixels (in directions.mif, FD.mif, etc) + # number of fixels in each voxel; by index.mif definition + count_vol = index_data[..., 0].astype(np.uint32) + # index of the first fixel in this voxel, in the list of all fixels + # (in directions.mif, FD.mif, etc) + id_vol = index_data[..., 1] max_id = id_vol.max() - max_fixel_id = max_id + int( - count_vol[id_vol == max_id] - ) # = the maximum id of fixels + 1 = # of fixels in entire image + # = the maximum id of fixels + 1 = # of fixels in entire image + max_fixel_id = max_id + int(count_vol[id_vol == max_id]) voxel_mask = count_vol > 0 # voxels that contains fixel(s), =1 masked_ids = id_vol[voxel_mask] # 1D array, len = # of voxels with fixel(s), value see id_vol masked_counts = count_vol[voxel_mask] # dim as masked_ids; value see count_vol - id_sort = np.argsort( - masked_ids - ) # indices that would sort array masked_ids value (i.e. first fixel's id in this voxel) from lowest to highest; so it's sorting voxels by their first fixel id + # indices that would sort array masked_ids value (i.e. first fixel's id in this voxel) from + # lowest to highest; so it's sorting voxels by their first fixel id + id_sort = np.argsort(masked_ids) sorted_counts = masked_counts[id_sort] - voxel_coords = np.column_stack( - np.nonzero(count_vol) - ) # dim: [# of voxels with fixel(s)] x 3, each row is the subscript i.e. (i,j,k) in 3D image of a voxel with fixel + # dim: [# of voxels with fixel(s)] x 3, each row is the subscript i.e. (i,j,k) in 3D + # image of a voxel with fixel + voxel_coords = np.column_stack(np.nonzero(count_vol)) fixel_id = 0 fixel_ids = np.arange(max_fixel_id, dtype=np.int32) fixel_voxel_ids = np.zeros_like(fixel_ids) for voxel_id, fixel_count in enumerate(sorted_counts): for _ in range(fixel_count): - fixel_voxel_ids[fixel_id] = ( - voxel_id # fixel_voxel_ids: 1D, len = # of fixels; each value is the voxel_id of the voxel where this fixel locates - ) + # fixel_voxel_ids: 1D, len = # of fixels; each value is the voxel_id of the voxel + # where this fixel locates + fixel_voxel_ids[fixel_id] = voxel_id fixel_id += 1 sorted_coords = voxel_coords[id_sort] diff --git a/src/modelarrayio/utils/voxels.py b/src/modelarrayio/utils/voxels.py index d3a1e16..efaa882 100644 --- a/src/modelarrayio/utils/voxels.py +++ b/src/modelarrayio/utils/voxels.py @@ -1,6 +1,5 @@ """Utility functions for voxel-wise data.""" -import os.path as op from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed @@ -8,10 +7,10 @@ import numpy as np from tqdm import tqdm -from modelarrayio.utils.s3_utils import is_s3_path, load_nibabel +from modelarrayio.utils.s3_utils import load_nibabel -def _load_cohort_voxels(cohort_df, group_mask_matrix, relative_root, s3_workers): +def _load_cohort_voxels(cohort_df, group_mask_matrix, s3_workers): """Load all voxel rows from the cohort, optionally in parallel. When s3_workers > 1, a ThreadPoolExecutor is used. Threads share memory so @@ -36,9 +35,7 @@ def _load_cohort_voxels(cohort_df, group_mask_matrix, relative_root, s3_workers) scalar_subj_counter[sn] += 1 src = row['source_file'] msk = row['source_mask_file'] - scalar_path = src if is_s3_path(src) else op.join(relative_root, src) - mask_path = msk if is_s3_path(msk) else op.join(relative_root, msk) - jobs.append((sn, subj_idx, scalar_path, mask_path)) + jobs.append((sn, subj_idx, src, msk)) sources_lists[sn].append(src) def _worker(job): @@ -79,6 +76,5 @@ def flattened_image(scalar_image, scalar_mask, group_mask_matrix): scalar_matrix = scalar_img.get_fdata() scalar_matrix[np.logical_not(scalar_mask_matrix)] = np.nan - return ( - scalar_matrix[group_mask_matrix].squeeze() - ) # .shape = (#voxels,) # squeeze() is to remove the 2nd dimension which is not necessary + # .shape = (#voxels,) # squeeze() is to remove the 2nd dimension which is not necessary + return scalar_matrix[group_mask_matrix].squeeze() diff --git a/test/test_cifti_cli.py b/test/test_cifti_cli.py index c6658a0..feb6bd2 100644 --- a/test/test_cifti_cli.py +++ b/test/test_cifti_cli.py @@ -59,11 +59,9 @@ def test_concifti_cli_creates_expected_hdf5(tmp_path, monkeypatch): [ 'concifti', '--cohort-file', - str(cohort_csv.name), - '--relative-root', - str(tmp_path), + str(cohort_csv), '--output-hdf5', - str(out_h5.name), + str(out_h5), '--backend', 'hdf5', '--dtype', @@ -72,8 +70,6 @@ def test_concifti_cli_creates_expected_hdf5(tmp_path, monkeypatch): 'gzip', '--compression-level', '1', - '--chunk-voxels', - '0', '--target-chunk-mb', '1.0', ], diff --git a/test/test_parser_utils.py b/test/test_parser_utils.py index fc8ea1e..1657fcf 100644 --- a/test/test_parser_utils.py +++ b/test/test_parser_utils.py @@ -9,7 +9,6 @@ def _parser_with_cohort() -> argparse.ArgumentParser: p = argparse.ArgumentParser() - parser_utils.add_relative_root_arg(p) parser_utils.add_output_hdf5_arg(p) parser_utils.add_cohort_arg(p) parser_utils.add_storage_args(p) @@ -18,10 +17,14 @@ def _parser_with_cohort() -> argparse.ArgumentParser: return p -def test_minimal_invocation_defaults() -> None: +def test_minimal_invocation_defaults(tmp_path_factory) -> None: + tmp_path = tmp_path_factory.mktemp('test_minimal_invocation_defaults') + cohort_file = tmp_path / 'cohort.csv' + cohort_file.write_text('subject_id,path\n1,path1\n2,path2') + p = _parser_with_cohort() - args = p.parse_args(['--cohort-file', 'cohort.csv']) - assert args.cohort_file == 'cohort.csv' + args = p.parse_args(['--cohort-file', str(cohort_file)]) + assert args.cohort_file == cohort_file assert args.storage_dtype == 'float32' assert args.compression == 'gzip' assert args.compression_level == 4 @@ -32,12 +35,16 @@ def test_minimal_invocation_defaults() -> None: assert args.log_level == 'INFO' -def test_storage_aliases_and_no_shuffle() -> None: +def test_storage_aliases_and_no_shuffle(tmp_path_factory) -> None: + tmp_path = tmp_path_factory.mktemp('test_storage_aliases_and_no_shuffle') + cohort_file = tmp_path / 'cohort.csv' + cohort_file.write_text('subject_id,path\n1,path1\n2,path2') + p = _parser_with_cohort() args = p.parse_args( [ '--cohort-file', - 'c.csv', + str(cohort_file), '--dtype', 'float64', '--compression', @@ -73,9 +80,13 @@ def test_tiledb_args_group() -> None: assert args.tdb_tile_voxels == 0 -def test_scalar_columns_optional() -> None: +def test_scalar_columns_optional(tmp_path_factory) -> None: + tmp_path = tmp_path_factory.mktemp('test_scalar_columns_optional') + cohort_file = tmp_path / 'cohort.csv' + cohort_file.write_text('subject_id,path\n1,path1\n2,path2') + p = argparse.ArgumentParser() parser_utils.add_cohort_arg(p) parser_utils.add_scalar_columns_arg(p) - args = p.parse_args(['--cohort-file', 'c.csv', '--scalar-columns', 'c1', 'c2']) + args = p.parse_args(['--cohort-file', str(cohort_file), '--scalar-columns', 'c1', 'c2']) assert args.scalar_columns == ['c1', 'c2'] diff --git a/test/test_voxels_cli.py b/test/test_voxels_cli.py index 278dfb8..3e4313b 100644 --- a/test/test_voxels_cli.py +++ b/test/test_voxels_cli.py @@ -6,7 +6,7 @@ import nibabel as nb import numpy as np -from modelarrayio.cli.voxels_to_h5 import main as convoxel_main +from modelarrayio.cli.nifti_to_h5 import main as convoxel_main def _make_nifti(data, affine=None): @@ -78,13 +78,11 @@ def test_convoxel_cli_creates_expected_hdf5(tmp_path, monkeypatch): [ 'convoxel', '--group-mask-file', - str(group_mask_file.name), + str(group_mask_file), '--cohort-file', - str(cohort_csv.name), - '--relative-root', - str(tmp_path), + str(cohort_csv), '--output-hdf5', - str(out_h5.name), + str(out_h5), '--backend', 'hdf5', '--dtype', @@ -93,8 +91,6 @@ def test_convoxel_cli_creates_expected_hdf5(tmp_path, monkeypatch): 'gzip', '--compression-level', '1', - '--chunk-voxels', - '0', '--target-chunk-mb', '1.0', ], diff --git a/test/test_voxels_s3.py b/test/test_voxels_s3.py index aaffb62..548c7d7 100644 --- a/test/test_voxels_s3.py +++ b/test/test_voxels_s3.py @@ -15,7 +15,7 @@ import numpy as np import pytest -from modelarrayio.cli.voxels_to_h5 import main as convoxel_main +from modelarrayio.cli.nifti_to_h5 import main as convoxel_main # Four confirmed ABIDE OHSU subjects used as test data OHSU_SUBJECTS = [ @@ -61,10 +61,9 @@ def test_convoxel_s3_parallel(tmp_path, group_mask_path, monkeypatch): """convoxel downloads s3:// paths in parallel and produces a valid HDF5.""" pytest.importorskip('boto3') - # Copy the group mask into tmp_path so --relative-root resolves it shutil.copy(group_mask_path, tmp_path / 'group_mask.nii.gz') - # Cohort CSV with s3:// paths — relative_root is not prepended to s3:// URIs + # Cohort CSV with s3:// paths cohort_csv = tmp_path / 'cohort.csv' with cohort_csv.open('w', newline='') as f: writer = csv.DictWriter(f, fieldnames=['scalar_name', 'source_file', 'source_mask_file']) @@ -90,10 +89,8 @@ def test_convoxel_s3_parallel(tmp_path, group_mask_path, monkeypatch): 'group_mask.nii.gz', '--cohort-file', str(cohort_csv), - '--relative-root', - str(tmp_path), '--output-hdf5', - str(out_h5.name), + str(out_h5), '--backend', 'hdf5', '--dtype', @@ -149,11 +146,9 @@ def test_convoxel_s3_serial_matches_parallel(tmp_path, group_mask_path, monkeypa base_argv = [ 'convoxel', '--group-mask-file', - 'group_mask.nii.gz', + str(group_mask_path), '--cohort-file', str(cohort_csv), - '--relative-root', - str(tmp_path), '--backend', 'hdf5', '--dtype', diff --git a/uv.lock b/uv.lock index bb40d7d..836c1d6 100644 --- a/uv.lock +++ b/uv.lock @@ -40,6 +40,7 @@ dependencies = [ { name = "jmespath" }, { name = "s3transfer" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/74/ec/636ab2aa7ad9e6bf6e297240ac2d44dba63cc6611e2d5038db318436d449/boto3-1.42.74.tar.gz", hash = "sha256:dbacd808cf2a3dadbf35f3dbd8de97b94dc9f78b1ebd439f38f552e0f9753577", size = 112739, upload-time = "2026-03-23T19:34:09.815Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ad/16/a264b4da2af99f4a12609b93fea941cce5ec41da14b33ed3fef77a910f0c/boto3-1.42.74-py3-none-any.whl", hash = "sha256:4bf89c044d618fe4435af854ab820f09dd43569c0df15d7beb0398f50b9aa970", size = 140557, upload-time = "2026-03-23T19:34:07.084Z" }, ] @@ -574,6 +575,7 @@ all = [ { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "sphinx-argparse" }, { name = "sphinx-copybutton" }, + { name = "sphinx-gallery" }, { name = "sphinx-rtd-theme" }, { name = "sphinxcontrib-apidoc" }, { name = "sphinxcontrib-bibtex" }, @@ -584,6 +586,7 @@ doc = [ { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "sphinx-argparse" }, { name = "sphinx-copybutton" }, + { name = "sphinx-gallery" }, { name = "sphinx-rtd-theme" }, { name = "sphinxcontrib-apidoc" }, { name = "sphinxcontrib-bibtex" }, @@ -631,6 +634,8 @@ requires-dist = [ { name = "sphinx-argparse", marker = "extra == 'doc'" }, { name = "sphinx-copybutton", marker = "extra == 'all'" }, { name = "sphinx-copybutton", marker = "extra == 'doc'" }, + { name = "sphinx-gallery", marker = "extra == 'all'" }, + { name = "sphinx-gallery", marker = "extra == 'doc'" }, { name = "sphinx-rtd-theme", marker = "extra == 'all'" }, { name = "sphinx-rtd-theme", marker = "extra == 'doc'" }, { name = "sphinxcontrib-apidoc", marker = "extra == 'all'" }, @@ -844,6 +849,93 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/db/61efa0d08a99f897ef98256b03e563092d36cc38dc4ebe4a85020fe40b31/pbr-7.0.3-py2.py3-none-any.whl", hash = "sha256:ff223894eb1cd271a98076b13d3badff3bb36c424074d26334cd25aebeecea6b", size = 131898, upload-time = "2025-11-03T17:04:54.875Z" }, ] +[[package]] +name = "pillow" +version = "12.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/42/5c74462b4fd957fcd7b13b04fb3205ff8349236ea74c7c375766d6c82288/pillow-12.1.1.tar.gz", hash = "sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4", size = 46980264, upload-time = "2026-02-11T04:23:07.146Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/46/5da1ec4a5171ee7bf1a0efa064aba70ba3d6e0788ce3f5acd1375d23c8c0/pillow-12.1.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e879bb6cd5c73848ef3b2b48b8af9ff08c5b71ecda8048b7dd22d8a33f60be32", size = 5304084, upload-time = "2026-02-11T04:20:27.501Z" }, + { url = "https://files.pythonhosted.org/packages/78/93/a29e9bc02d1cf557a834da780ceccd54e02421627200696fcf805ebdc3fb/pillow-12.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:365b10bb9417dd4498c0e3b128018c4a624dc11c7b97d8cc54effe3b096f4c38", size = 4657866, upload-time = "2026-02-11T04:20:29.827Z" }, + { url = "https://files.pythonhosted.org/packages/13/84/583a4558d492a179d31e4aae32eadce94b9acf49c0337c4ce0b70e0a01f2/pillow-12.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d4ce8e329c93845720cd2014659ca67eac35f6433fd3050393d85f3ecef0dad5", size = 6232148, upload-time = "2026-02-11T04:20:31.329Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e2/53c43334bbbb2d3b938978532fbda8e62bb6e0b23a26ce8592f36bcc4987/pillow-12.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc354a04072b765eccf2204f588a7a532c9511e8b9c7f900e1b64e3e33487090", size = 8038007, upload-time = "2026-02-11T04:20:34.225Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/3d0e79c8a9d58150dd98e199d7c1c56861027f3829a3a60b3c2784190180/pillow-12.1.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e7976bf1910a8116b523b9f9f58bf410f3e8aa330cd9a2bb2953f9266ab49af", size = 6345418, upload-time = "2026-02-11T04:20:35.858Z" }, + { url = "https://files.pythonhosted.org/packages/a2/c8/46dfeac5825e600579157eea177be43e2f7ff4a99da9d0d0a49533509ac5/pillow-12.1.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:597bd9c8419bc7c6af5604e55847789b69123bbe25d65cc6ad3012b4f3c98d8b", size = 7034590, upload-time = "2026-02-11T04:20:37.91Z" }, + { url = "https://files.pythonhosted.org/packages/af/bf/e6f65d3db8a8bbfeaf9e13cc0417813f6319863a73de934f14b2229ada18/pillow-12.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2c1fc0f2ca5f96a3c8407e41cca26a16e46b21060fe6d5b099d2cb01412222f5", size = 6458655, upload-time = "2026-02-11T04:20:39.496Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c2/66091f3f34a25894ca129362e510b956ef26f8fb67a0e6417bc5744e56f1/pillow-12.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:578510d88c6229d735855e1f278aa305270438d36a05031dfaae5067cc8eb04d", size = 7159286, upload-time = "2026-02-11T04:20:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5a/24bc8eb526a22f957d0cec6243146744966d40857e3d8deb68f7902ca6c1/pillow-12.1.1-cp311-cp311-win32.whl", hash = "sha256:7311c0a0dcadb89b36b7025dfd8326ecfa36964e29913074d47382706e516a7c", size = 6328663, upload-time = "2026-02-11T04:20:43.184Z" }, + { url = "https://files.pythonhosted.org/packages/31/03/bef822e4f2d8f9d7448c133d0a18185d3cce3e70472774fffefe8b0ed562/pillow-12.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:fbfa2a7c10cc2623f412753cddf391c7f971c52ca40a3f65dc5039b2939e8563", size = 7031448, upload-time = "2026-02-11T04:20:44.696Z" }, + { url = "https://files.pythonhosted.org/packages/49/70/f76296f53610bd17b2e7d31728b8b7825e3ac3b5b3688b51f52eab7c0818/pillow-12.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:b81b5e3511211631b3f672a595e3221252c90af017e399056d0faabb9538aa80", size = 2453651, upload-time = "2026-02-11T04:20:46.243Z" }, + { url = "https://files.pythonhosted.org/packages/07/d3/8df65da0d4df36b094351dce696f2989bec731d4f10e743b1c5f4da4d3bf/pillow-12.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab323b787d6e18b3d91a72fc99b1a2c28651e4358749842b8f8dfacd28ef2052", size = 5262803, upload-time = "2026-02-11T04:20:47.653Z" }, + { url = "https://files.pythonhosted.org/packages/d6/71/5026395b290ff404b836e636f51d7297e6c83beceaa87c592718747e670f/pillow-12.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:adebb5bee0f0af4909c30db0d890c773d1a92ffe83da908e2e9e720f8edf3984", size = 4657601, upload-time = "2026-02-11T04:20:49.328Z" }, + { url = "https://files.pythonhosted.org/packages/b1/2e/1001613d941c67442f745aff0f7cc66dd8df9a9c084eb497e6a543ee6f7e/pillow-12.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb66b7cc26f50977108790e2456b7921e773f23db5630261102233eb355a3b79", size = 6234995, upload-time = "2026-02-11T04:20:51.032Z" }, + { url = "https://files.pythonhosted.org/packages/07/26/246ab11455b2549b9233dbd44d358d033a2f780fa9007b61a913c5b2d24e/pillow-12.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee2810642b2898bb187ced9b349e95d2a7272930796e022efaf12e99dccd293", size = 8045012, upload-time = "2026-02-11T04:20:52.882Z" }, + { url = "https://files.pythonhosted.org/packages/b2/8b/07587069c27be7535ac1fe33874e32de118fbd34e2a73b7f83436a88368c/pillow-12.1.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0b1cd6232e2b618adcc54d9882e4e662a089d5768cd188f7c245b4c8c44a397", size = 6349638, upload-time = "2026-02-11T04:20:54.444Z" }, + { url = "https://files.pythonhosted.org/packages/ff/79/6df7b2ee763d619cda2fb4fea498e5f79d984dae304d45a8999b80d6cf5c/pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7aac39bcf8d4770d089588a2e1dd111cbaa42df5a94be3114222057d68336bd0", size = 7041540, upload-time = "2026-02-11T04:20:55.97Z" }, + { url = "https://files.pythonhosted.org/packages/2c/5e/2ba19e7e7236d7529f4d873bdaf317a318896bac289abebd4bb00ef247f0/pillow-12.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ab174cd7d29a62dd139c44bf74b698039328f45cb03b4596c43473a46656b2f3", size = 6462613, upload-time = "2026-02-11T04:20:57.542Z" }, + { url = "https://files.pythonhosted.org/packages/03/03/31216ec124bb5c3dacd74ce8efff4cc7f52643653bad4825f8f08c697743/pillow-12.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:339ffdcb7cbeaa08221cd401d517d4b1fe7a9ed5d400e4a8039719238620ca35", size = 7166745, upload-time = "2026-02-11T04:20:59.196Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e7/7c4552d80052337eb28653b617eafdef39adfb137c49dd7e831b8dc13bc5/pillow-12.1.1-cp312-cp312-win32.whl", hash = "sha256:5d1f9575a12bed9e9eedd9a4972834b08c97a352bd17955ccdebfeca5913fa0a", size = 6328823, upload-time = "2026-02-11T04:21:01.385Z" }, + { url = "https://files.pythonhosted.org/packages/3d/17/688626d192d7261bbbf98846fc98995726bddc2c945344b65bec3a29d731/pillow-12.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:21329ec8c96c6e979cd0dfd29406c40c1d52521a90544463057d2aaa937d66a6", size = 7033367, upload-time = "2026-02-11T04:21:03.536Z" }, + { url = "https://files.pythonhosted.org/packages/ed/fe/a0ef1f73f939b0eca03ee2c108d0043a87468664770612602c63266a43c4/pillow-12.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:af9a332e572978f0218686636610555ae3defd1633597be015ed50289a03c523", size = 2453811, upload-time = "2026-02-11T04:21:05.116Z" }, + { url = "https://files.pythonhosted.org/packages/d5/11/6db24d4bd7685583caeae54b7009584e38da3c3d4488ed4cd25b439de486/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e", size = 4062689, upload-time = "2026-02-11T04:21:06.804Z" }, + { url = "https://files.pythonhosted.org/packages/33/c0/ce6d3b1fe190f0021203e0d9b5b99e57843e345f15f9ef22fcd43842fd21/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9", size = 4138535, upload-time = "2026-02-11T04:21:08.452Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c6/d5eb6a4fb32a3f9c21a8c7613ec706534ea1cf9f4b3663e99f0d83f6fca8/pillow-12.1.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6", size = 3601364, upload-time = "2026-02-11T04:21:10.194Z" }, + { url = "https://files.pythonhosted.org/packages/14/a1/16c4b823838ba4c9c52c0e6bbda903a3fe5a1bdbf1b8eb4fff7156f3e318/pillow-12.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60", size = 5262561, upload-time = "2026-02-11T04:21:11.742Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ad/ad9dc98ff24f485008aa5cdedaf1a219876f6f6c42a4626c08bc4e80b120/pillow-12.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2", size = 4657460, upload-time = "2026-02-11T04:21:13.786Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f1a4ea9a895b5732152789326202a82464d5254759fbacae4deea3069334/pillow-12.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850", size = 6232698, upload-time = "2026-02-11T04:21:15.949Z" }, + { url = "https://files.pythonhosted.org/packages/95/f4/86f51b8745070daf21fd2e5b1fe0eb35d4db9ca26e6d58366562fb56a743/pillow-12.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289", size = 8041706, upload-time = "2026-02-11T04:21:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/29/9b/d6ecd956bb1266dd1045e995cce9b8d77759e740953a1c9aad9502a0461e/pillow-12.1.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e", size = 6346621, upload-time = "2026-02-11T04:21:19.547Z" }, + { url = "https://files.pythonhosted.org/packages/71/24/538bff45bde96535d7d998c6fed1a751c75ac7c53c37c90dc2601b243893/pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717", size = 7038069, upload-time = "2026-02-11T04:21:21.378Z" }, + { url = "https://files.pythonhosted.org/packages/94/0e/58cb1a6bc48f746bc4cb3adb8cabff73e2742c92b3bf7a220b7cf69b9177/pillow-12.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a", size = 6460040, upload-time = "2026-02-11T04:21:23.148Z" }, + { url = "https://files.pythonhosted.org/packages/6c/57/9045cb3ff11eeb6c1adce3b2d60d7d299d7b273a2e6c8381a524abfdc474/pillow-12.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029", size = 7164523, upload-time = "2026-02-11T04:21:25.01Z" }, + { url = "https://files.pythonhosted.org/packages/73/f2/9be9cb99f2175f0d4dbadd6616ce1bf068ee54a28277ea1bf1fbf729c250/pillow-12.1.1-cp313-cp313-win32.whl", hash = "sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b", size = 6332552, upload-time = "2026-02-11T04:21:27.238Z" }, + { url = "https://files.pythonhosted.org/packages/3f/eb/b0834ad8b583d7d9d42b80becff092082a1c3c156bb582590fcc973f1c7c/pillow-12.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1", size = 7040108, upload-time = "2026-02-11T04:21:29.462Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/fc09634e2aabdd0feabaff4a32f4a7d97789223e7c2042fd805ea4b4d2c2/pillow-12.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a", size = 2453712, upload-time = "2026-02-11T04:21:31.072Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/b9d62794fc8a0dd14c1943df68347badbd5511103e0d04c035ffe5cf2255/pillow-12.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da", size = 5264880, upload-time = "2026-02-11T04:21:32.865Z" }, + { url = "https://files.pythonhosted.org/packages/26/9d/e03d857d1347fa5ed9247e123fcd2a97b6220e15e9cb73ca0a8d91702c6e/pillow-12.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc", size = 4660616, upload-time = "2026-02-11T04:21:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ec/8a6d22afd02570d30954e043f09c32772bfe143ba9285e2fdb11284952cd/pillow-12.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c", size = 6269008, upload-time = "2026-02-11T04:21:36.623Z" }, + { url = "https://files.pythonhosted.org/packages/3d/1d/6d875422c9f28a4a361f495a5f68d9de4a66941dc2c619103ca335fa6446/pillow-12.1.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8", size = 8073226, upload-time = "2026-02-11T04:21:38.585Z" }, + { url = "https://files.pythonhosted.org/packages/a1/cd/134b0b6ee5eda6dc09e25e24b40fdafe11a520bc725c1d0bbaa5e00bf95b/pillow-12.1.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20", size = 6380136, upload-time = "2026-02-11T04:21:40.562Z" }, + { url = "https://files.pythonhosted.org/packages/7a/a9/7628f013f18f001c1b98d8fffe3452f306a70dc6aba7d931019e0492f45e/pillow-12.1.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13", size = 7067129, upload-time = "2026-02-11T04:21:42.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f8/66ab30a2193b277785601e82ee2d49f68ea575d9637e5e234faaa98efa4c/pillow-12.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf", size = 6491807, upload-time = "2026-02-11T04:21:44.22Z" }, + { url = "https://files.pythonhosted.org/packages/da/0b/a877a6627dc8318fdb84e357c5e1a758c0941ab1ddffdafd231983788579/pillow-12.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524", size = 7190954, upload-time = "2026-02-11T04:21:46.114Z" }, + { url = "https://files.pythonhosted.org/packages/83/43/6f732ff85743cf746b1361b91665d9f5155e1483817f693f8d57ea93147f/pillow-12.1.1-cp313-cp313t-win32.whl", hash = "sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986", size = 6336441, upload-time = "2026-02-11T04:21:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/3b/44/e865ef3986611bb75bfabdf94a590016ea327833f434558801122979cd0e/pillow-12.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c", size = 7045383, upload-time = "2026-02-11T04:21:50.015Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c6/f4fb24268d0c6908b9f04143697ea18b0379490cb74ba9e8d41b898bd005/pillow-12.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3", size = 2456104, upload-time = "2026-02-11T04:21:51.633Z" }, + { url = "https://files.pythonhosted.org/packages/03/d0/bebb3ffbf31c5a8e97241476c4cf8b9828954693ce6744b4a2326af3e16b/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af", size = 4062652, upload-time = "2026-02-11T04:21:53.19Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c0/0e16fb0addda4851445c28f8350d8c512f09de27bbb0d6d0bbf8b6709605/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f", size = 4138823, upload-time = "2026-02-11T04:22:03.088Z" }, + { url = "https://files.pythonhosted.org/packages/6b/fb/6170ec655d6f6bb6630a013dd7cf7bc218423d7b5fa9071bf63dc32175ae/pillow-12.1.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642", size = 3601143, upload-time = "2026-02-11T04:22:04.909Z" }, + { url = "https://files.pythonhosted.org/packages/59/04/dc5c3f297510ba9a6837cbb318b87dd2b8f73eb41a43cc63767f65cb599c/pillow-12.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd", size = 5266254, upload-time = "2026-02-11T04:22:07.656Z" }, + { url = "https://files.pythonhosted.org/packages/05/30/5db1236b0d6313f03ebf97f5e17cda9ca060f524b2fcc875149a8360b21c/pillow-12.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202", size = 4657499, upload-time = "2026-02-11T04:22:09.613Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/008d2ca0eb612e81968e8be0bbae5051efba24d52debf930126d7eaacbba/pillow-12.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f", size = 6232137, upload-time = "2026-02-11T04:22:11.434Z" }, + { url = "https://files.pythonhosted.org/packages/70/f1/f14d5b8eeb4b2cd62b9f9f847eb6605f103df89ef619ac68f92f748614ea/pillow-12.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f", size = 8042721, upload-time = "2026-02-11T04:22:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d6/17824509146e4babbdabf04d8171491fa9d776f7061ff6e727522df9bd03/pillow-12.1.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f", size = 6347798, upload-time = "2026-02-11T04:22:15.449Z" }, + { url = "https://files.pythonhosted.org/packages/d1/ee/c85a38a9ab92037a75615aba572c85ea51e605265036e00c5b67dfafbfe2/pillow-12.1.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e", size = 7039315, upload-time = "2026-02-11T04:22:17.24Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f3/bc8ccc6e08a148290d7523bde4d9a0d6c981db34631390dc6e6ec34cacf6/pillow-12.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0", size = 6462360, upload-time = "2026-02-11T04:22:19.111Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ab/69a42656adb1d0665ab051eec58a41f169ad295cf81ad45406963105408f/pillow-12.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb", size = 7165438, upload-time = "2026-02-11T04:22:21.041Z" }, + { url = "https://files.pythonhosted.org/packages/02/46/81f7aa8941873f0f01d4b55cc543b0a3d03ec2ee30d617a0448bf6bd6dec/pillow-12.1.1-cp314-cp314-win32.whl", hash = "sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f", size = 6431503, upload-time = "2026-02-11T04:22:22.833Z" }, + { url = "https://files.pythonhosted.org/packages/40/72/4c245f7d1044b67affc7f134a09ea619d4895333d35322b775b928180044/pillow-12.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15", size = 7176748, upload-time = "2026-02-11T04:22:24.64Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ad/8a87bdbe038c5c698736e3348af5c2194ffb872ea52f11894c95f9305435/pillow-12.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f", size = 2544314, upload-time = "2026-02-11T04:22:26.685Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/efd18493f9de13b87ede7c47e69184b9e859e4427225ea962e32e56a49bc/pillow-12.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8", size = 5268612, upload-time = "2026-02-11T04:22:29.884Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f1/4f42eb2b388eb2ffc660dcb7f7b556c1015c53ebd5f7f754965ef997585b/pillow-12.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9", size = 4660567, upload-time = "2026-02-11T04:22:31.799Z" }, + { url = "https://files.pythonhosted.org/packages/01/54/df6ef130fa43e4b82e32624a7b821a2be1c5653a5fdad8469687a7db4e00/pillow-12.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60", size = 6269951, upload-time = "2026-02-11T04:22:33.921Z" }, + { url = "https://files.pythonhosted.org/packages/a9/48/618752d06cc44bb4aae8ce0cd4e6426871929ed7b46215638088270d9b34/pillow-12.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7", size = 8074769, upload-time = "2026-02-11T04:22:35.877Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bd/f1d71eb39a72fa088d938655afba3e00b38018d052752f435838961127d8/pillow-12.1.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f", size = 6381358, upload-time = "2026-02-11T04:22:37.698Z" }, + { url = "https://files.pythonhosted.org/packages/64/ef/c784e20b96674ed36a5af839305f55616f8b4f8aa8eeccf8531a6e312243/pillow-12.1.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586", size = 7068558, upload-time = "2026-02-11T04:22:39.597Z" }, + { url = "https://files.pythonhosted.org/packages/73/cb/8059688b74422ae61278202c4e1ad992e8a2e7375227be0a21c6b87ca8d5/pillow-12.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce", size = 6493028, upload-time = "2026-02-11T04:22:42.73Z" }, + { url = "https://files.pythonhosted.org/packages/c6/da/e3c008ed7d2dd1f905b15949325934510b9d1931e5df999bb15972756818/pillow-12.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8", size = 7191940, upload-time = "2026-02-11T04:22:44.543Z" }, + { url = "https://files.pythonhosted.org/packages/01/4a/9202e8d11714c1fc5951f2e1ef362f2d7fbc595e1f6717971d5dd750e969/pillow-12.1.1-cp314-cp314t-win32.whl", hash = "sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36", size = 6438736, upload-time = "2026-02-11T04:22:46.347Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ca/cbce2327eb9885476b3957b2e82eb12c866a8b16ad77392864ad601022ce/pillow-12.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b", size = 7182894, upload-time = "2026-02-11T04:22:48.114Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d2/de599c95ba0a973b94410477f8bf0b6f0b5e67360eb89bcb1ad365258beb/pillow-12.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334", size = 2546446, upload-time = "2026-02-11T04:22:50.342Z" }, + { url = "https://files.pythonhosted.org/packages/56/11/5d43209aa4cb58e0cc80127956ff1796a68b928e6324bbf06ef4db34367b/pillow-12.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:600fd103672b925fe62ed08e0d874ea34d692474df6f4bf7ebe148b30f89f39f", size = 5228606, upload-time = "2026-02-11T04:22:52.106Z" }, + { url = "https://files.pythonhosted.org/packages/5f/d5/3b005b4e4fda6698b371fa6c21b097d4707585d7db99e98d9b0b87ac612a/pillow-12.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:665e1b916b043cef294bc54d47bf02d87e13f769bc4bc5fa225a24b3a6c5aca9", size = 4622321, upload-time = "2026-02-11T04:22:53.827Z" }, + { url = "https://files.pythonhosted.org/packages/df/36/ed3ea2d594356fd8037e5a01f6156c74bc8d92dbb0fa60746cc96cabb6e8/pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:495c302af3aad1ca67420ddd5c7bd480c8867ad173528767d906428057a11f0e", size = 5247579, upload-time = "2026-02-11T04:22:56.094Z" }, + { url = "https://files.pythonhosted.org/packages/54/9a/9cc3e029683cf6d20ae5085da0dafc63148e3252c2f13328e553aaa13cfb/pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8fd420ef0c52c88b5a035a0886f367748c72147b2b8f384c9d12656678dfdfa9", size = 6989094, upload-time = "2026-02-11T04:22:58.288Z" }, + { url = "https://files.pythonhosted.org/packages/00/98/fc53ab36da80b88df0967896b6c4b4cd948a0dc5aa40a754266aa3ae48b3/pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f975aa7ef9684ce7e2c18a3aa8f8e2106ce1e46b94ab713d156b2898811651d3", size = 5313850, upload-time = "2026-02-11T04:23:00.554Z" }, + { url = "https://files.pythonhosted.org/packages/30/02/00fa585abfd9fe9d73e5f6e554dc36cc2b842898cbfc46d70353dae227f8/pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8089c852a56c2966cf18835db62d9b34fef7ba74c726ad943928d494fa7f4735", size = 5963343, upload-time = "2026-02-11T04:23:02.934Z" }, + { url = "https://files.pythonhosted.org/packages/f2/26/c56ce33ca856e358d27fda9676c055395abddb82c35ac0f593877ed4562e/pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e", size = 7029880, upload-time = "2026-02-11T04:23:04.783Z" }, +] + [[package]] name = "platformdirs" version = "4.9.4" @@ -1242,6 +1334,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, ] +[[package]] +name = "sphinx-gallery" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pillow" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/14/9238ac61932299b38c20c7c37dbfe60348c0348ea4d400f9ef25875b3bf7/sphinx_gallery-0.20.0.tar.gz", hash = "sha256:70281510c6183d812d3595957005ccf555c5a793f207410f6cd16a25bf08d735", size = 473502, upload-time = "2025-12-02T15:51:37.277Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/fd/818a53d4da56ef2da7b08f77bb3a825635941d1fcc6b6a490995dec1a81c/sphinx_gallery-0.20.0-py3-none-any.whl", hash = "sha256:188b7456e269649945825661b76cdbfbf0b70c2cfd5b75c9a11fe52519879e4d", size = 458655, upload-time = "2025-12-02T15:51:35.311Z" }, +] + [[package]] name = "sphinx-rtd-theme" version = "3.1.0"