diff --git a/.github/workflows/macos_tests.yml b/.github/workflows/macos_tests.yml index c8df2737..726c9144 100644 --- a/.github/workflows/macos_tests.yml +++ b/.github/workflows/macos_tests.yml @@ -4,25 +4,25 @@ name: macos build on: [push, pull_request, workflow_dispatch] jobs: - build: - runs-on: ["macos-13"] + build: + runs-on: ["macos-latest"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install spare scores - run: | - python -m pip cache purge - pip install -r requirements.txt - pip install setuptools twine wheel - python -m pip install . - - name: Run unit tests - run: | - cd tests/ && pytest --cov=../ --cov-report=xml - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v4.0.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: CBICA/NiChart_DLMUSE + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install spare scores + run: | + python -m pip cache purge + pip install -r requirements.txt + pip install setuptools twine wheel + python -m pip install . + - name: Run unit tests + run: | + cd tests/ && pytest --cov=../ --cov-report=xml + - name: Upload Coverage to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: CBICA/NiChart_DLMUSE diff --git a/NiChart_DLMUSE/SegmentImage.py b/NiChart_DLMUSE/SegmentImage.py index ae043d6a..90b9adcd 100644 --- a/NiChart_DLMUSE/SegmentImage.py +++ b/NiChart_DLMUSE/SegmentImage.py @@ -2,8 +2,6 @@ import os import shutil from typing import Any -import DLMUSE -import DLICV def run_dlicv( diff --git a/NiChart_DLMUSE/__init__.py b/NiChart_DLMUSE/__init__.py index aa827483..1d89acb6 100644 --- a/NiChart_DLMUSE/__init__.py +++ b/NiChart_DLMUSE/__init__.py @@ -1 +1 @@ -from .dlmuse_pipeline import run_pipeline, run_dlicv, run_dlmuse +from .dlmuse_pipeline import run_dlicv, run_dlmuse, run_ndlmuse_pipeline diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index 026353a0..6b8a294b 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -6,21 +6,12 @@ """ import argparse -import os -import shutil -import threading -from .dlmuse_pipeline import run_pipeline -from .utils import ( - collect_T1, - merge_bids_output_data, - merge_output_data, - remove_subfolders, - split_data, -) +import pkg_resources # type: ignore -# VERSION = pkg_resources.require("NiChart_DLMUSE")[0].version -VERSION = "1.0.7" +from NiChart_DLMUSE.dlmuse_pipeline import run_ndlmuse_pipeline + +VERSION = pkg_resources.require("NiChart_DLMUSE")[0].version def main() -> None: @@ -74,8 +65,7 @@ def main() -> None: # DEVICE argument parser.add_argument( - "-d", - "--device", + "-device", type=str, help="Device (cpu, cuda, or mps)", default=None, @@ -137,99 +127,21 @@ def main() -> None: args = parser.parse_args() - in_dir = args.in_dir - out_dir = args.out_dir - device = args.device - dlicv_extra_args = args.dlicv_args - dlmuse_extra_args = args.dlmuse_args - print() print("Arguments:") print(args) print() - if not os.path.isdir(out_dir): - print(f"Can't find {out_dir}, creating it...") - # os.system(f"mkdir {out_dir}") - os.mkdir(out_dir) - - elif len(os.listdir(out_dir)) != 0: - print(f"Emptying output folder: {out_dir}...") - for root, dirs, files in os.walk(out_dir): - for f in files: - os.unlink(os.path.join(root, f)) - for d in dirs: - shutil.rmtree(os.path.join(root, d)) - - if args.clear_cache: - os.system("DLICV -i ./dummy -o ./dummy --clear_cache") - os.system("DLMUSE -i ./dummy -o ./dummy --clear_cache") - - working_dir = os.path.join(os.path.abspath(out_dir)) - - # Run pipeline - if args.bids is True: - if int(args.cores) > 1: - collect_T1(in_dir, out_dir) - - no_threads = int(args.cores) - subfolders = split_data("raw_temp_T1", no_threads) - threads = [] - for i in range(len(subfolders)): - curr_out_dir = out_dir + f"/split_{i}" - curr_thread = threading.Thread( - target=run_pipeline, - args=( - subfolders[i], - curr_out_dir, - device, - dlmuse_extra_args, - dlicv_extra_args, - i, - ), - ) - curr_thread.start() - threads.append(curr_thread) - - for t in threads: - t.join() - - merge_bids_output_data(working_dir) - remove_subfolders("raw_temp_T1") - remove_subfolders(out_dir) - else: # No core parallelization - run_pipeline(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) - - else: # Non-BIDS - if int(args.cores) > 1: - no_threads = int(args.cores) - subfolders = split_data(in_dir, no_threads) - - threads = [] - for i in range(len(subfolders)): - curr_out_dir = out_dir + f"/split_{i}" - curr_thread = threading.Thread( - target=run_pipeline, - args=( - subfolders[i], - curr_out_dir, - device, - dlmuse_extra_args, - dlicv_extra_args, - i, - ), - ) - curr_thread.start() - threads.append(curr_thread) - - for t in threads: - t.join() - - merge_output_data(out_dir) - remove_subfolders(in_dir) - remove_subfolders(out_dir) - else: # No core parallelization - run_pipeline(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) + run_ndlmuse_pipeline( + args.in_dir, + args.out_dir, + args.device, + args.dlicv_extra_args, + args.dlmuse_extra_args, + args.clear_cache, + args.bids, + args.cores, + ) if __name__ == "__main__": diff --git a/NiChart_DLMUSE/dlmuse_pipeline.py b/NiChart_DLMUSE/dlmuse_pipeline.py index 8268fd79..82183abf 100644 --- a/NiChart_DLMUSE/dlmuse_pipeline.py +++ b/NiChart_DLMUSE/dlmuse_pipeline.py @@ -1,14 +1,24 @@ import logging import os +import shutil +import threading +from typing import Any import pkg_resources # type: ignore -from .CalcROIVol import apply_create_roi_csv, combine_roi_csv -from .MaskImage import apply_combine_masks, apply_mask_img -from .RelabelROI import apply_relabel_rois -from .ReorientImage import apply_reorient_img, apply_reorient_to_init -from .SegmentImage import run_dlicv, run_dlmuse -from .utils import make_img_list +from NiChart_DLMUSE.CalcROIVol import apply_create_roi_csv, combine_roi_csv +from NiChart_DLMUSE.MaskImage import apply_combine_masks, apply_mask_img +from NiChart_DLMUSE.RelabelROI import apply_relabel_rois +from NiChart_DLMUSE.ReorientImage import apply_reorient_img, apply_reorient_to_init +from NiChart_DLMUSE.SegmentImage import run_dlicv, run_dlmuse +from NiChart_DLMUSE.utils import ( + collect_T1, + make_img_list, + merge_bids_output_data, + merge_output_data, + remove_subfolders, + split_data, +) # Config vars SUFF_LPS = "_LPS.nii.gz" @@ -35,17 +45,131 @@ logging.basicConfig(filename="pipeline.log", encoding="utf-8", level=logging.DEBUG) -def run_pipeline( +def run_ndlmuse_pipeline( + in_dir: str, + out_dir: str, + device: str, + dlicv_extra_args: str, + dlmuse_extra_args: str, + clear_cache: bool, + bids: bool, + cores: str, +) -> None: + """ + NiChart pipeline + + :param in_dir: The input directory + :type in_dir: str + :param out_dir: The output directory + :type out_dir: str + :type device: cpu/cuda/mps + :param device: str + :param dlicv_extra_args: Extra arguments for DLICV API + :type dlicv_extra_args: str + :param dlmuse_extra_args: Extra arguments for DLMUSE API + :type dlmuse_extra_args: str + :param clear_cache: True if cache should be cleared + :type clear_cache: bool + :param bids: True if your input is a bids type folder + :type bids: bool + :param cores: The number of cores(default is 4) + :type cores: str + + :rtype: None + """ + if not os.path.isdir(out_dir): + print(f"Can't find {out_dir}, creating it...") + # os.system(f"mkdir {out_dir}") + os.mkdir(out_dir) + + elif len(os.listdir(out_dir)) != 0: + print(f"Emptying output folder: {out_dir}...") + for root, dirs, files in os.walk(out_dir): + for f in files: + os.unlink(os.path.join(root, f)) + for d in dirs: + shutil.rmtree(os.path.join(root, d)) + if clear_cache: + os.system("DLICV -i ./dummy -o ./dummy --clear_cache") + os.system("DLMUSE -i ./dummy -o ./dummy --clear_cache") + + working_dir = os.path.join(os.path.abspath(out_dir)) + + if bids is True: + if int(cores) > 1: + collect_T1(in_dir, out_dir) + + no_threads = int(cores) + subfolders = split_data("raw_temp_T1", no_threads) + threads = [] + for i in range(len(subfolders)): + curr_out_dir = out_dir + f"/split_{i}" + curr_thread = threading.Thread( + target=run_thread, + args=( + subfolders[i], + curr_out_dir, + device, + dlmuse_extra_args, + dlicv_extra_args, + i, + ), + ) + curr_thread.start() + threads.append(curr_thread) + + for t in threads: + t.join() + + merge_bids_output_data(working_dir) + remove_subfolders("raw_temp_T1") + remove_subfolders(out_dir) + else: # No core parallelization + run_thread(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) + + else: # Non-BIDS + if int(cores) > 1: + no_threads = int(cores) + subfolders = split_data(in_dir, no_threads) + + threads = [] + for i in range(len(subfolders)): + curr_out_dir = out_dir + f"/split_{i}" + curr_thread = threading.Thread( + target=run_thread, + args=( + subfolders[i], + curr_out_dir, + device, + dlmuse_extra_args, + dlicv_extra_args, + i, + ), + ) + curr_thread.start() + threads.append(curr_thread) + + for t in threads: + t.join() + + merge_output_data(out_dir) + remove_subfolders(in_dir) + remove_subfolders(out_dir) + else: # No core parallelization + run_thread(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) + + +def run_thread( in_data: str, out_dir: str, device: str, - dlmuse_extra_args: str = '', - dlicv_extra_args: str = '', + dlmuse_extra_args: str = "", + dlicv_extra_args: str = "", sub_fldr: int = 1, - progress_bar = None, + progress_bar: Any = None, ) -> None: """ - NiChart pipeline + Run a thread of the pipeline :param in_data: the input directory :type in_data: str diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index f4c4e310..89c2a344 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -65,7 +65,9 @@ def remove_common_suffix(list_files: list) -> list: bnames = list_files if len(list_files) == 1: - if list_files[0].endswith('_T1'): # If there is a single image with suffix _T1, remove it + if list_files[0].endswith( + "_T1" + ): # If there is a single image with suffix _T1, remove it bnames = [x[0:-3] for x in bnames] return bnames @@ -347,7 +349,7 @@ def remove_subfolders(in_dir: str) -> None: def merge_output_data(in_dir: str) -> None: """ - Takes all the results from the temp_working_fir and moves them into + Takes all the results from the temp_working_dir and moves them into the output folder :param in_dir: the input directory diff --git a/requirements.txt b/requirements.txt index 79414b1a..4d118458 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Main pipeline -torch==2.2.1 +torch==2.3.1 DLICV DLMUSE nibabel>=5.2 @@ -12,3 +12,6 @@ huggingface_hub argparse pathlib pre-commit + +# testing +pandas diff --git a/setup.py b/setup.py index 9698ec15..5bb55bc4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ url="https://github.com/CBICA/NiChart_DLMUSE", python_requires=">=3.9", install_requires=[ - "torch<=2.2.1", + "torch<=2.3.1", "DLICV", "DLMUSE", "huggingface_hub", diff --git a/tests/test_utils.py b/tests/test_utils.py index f61e764e..e364ec25 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -58,7 +58,19 @@ def testing_make_img_list() -> None: def testing_get_bids_prefix() -> None: - pass + temp_file = "test-1234" + assert get_bids_prefix(temp_file) == "test" + temp_file = "test-1" + assert get_bids_prefix(temp_file) == "test" + temp_file = "test" + assert get_bids_prefix(temp_file) == "test" + + temp_folder = "test_1234" + assert get_bids_prefix(temp_folder, True) == "test" + temp_folder = "test_1" + assert get_bids_prefix(temp_folder, True) == "test" + temp_folder = "test" + assert get_bids_prefix(temp_folder, True) == "test" def testing_collect_T1() -> None: