From b6e80acf89c67386e980b3e5d5810ce8a326716e Mon Sep 17 00:00:00 2001 From: spirosmaggioros Date: Fri, 10 Jan 2025 00:14:07 +0000 Subject: [PATCH 1/7] [Update] Pipeline function for code usage instead of only CLI | Update torch version(tested) | Added test case for BIDS I/O --- NiChart_DLMUSE/SegmentImage.py | 2 - NiChart_DLMUSE/__init__.py | 2 +- NiChart_DLMUSE/__main__.py | 111 ++++-------------------- NiChart_DLMUSE/dlmuse_pipeline.py | 136 ++++++++++++++++++++++++++++-- NiChart_DLMUSE/utils.py | 6 +- requirements.txt | 5 +- setup.py | 2 +- tests/test_utils.py | 14 ++- 8 files changed, 169 insertions(+), 109 deletions(-) diff --git a/NiChart_DLMUSE/SegmentImage.py b/NiChart_DLMUSE/SegmentImage.py index ae043d6a..90b9adcd 100644 --- a/NiChart_DLMUSE/SegmentImage.py +++ b/NiChart_DLMUSE/SegmentImage.py @@ -2,8 +2,6 @@ import os import shutil from typing import Any -import DLMUSE -import DLICV def run_dlicv( diff --git a/NiChart_DLMUSE/__init__.py b/NiChart_DLMUSE/__init__.py index aa827483..36e71672 100644 --- a/NiChart_DLMUSE/__init__.py +++ b/NiChart_DLMUSE/__init__.py @@ -1 +1 @@ -from .dlmuse_pipeline import run_pipeline, run_dlicv, run_dlmuse +from .dlmuse_pipeline import run_dlicv, run_dlmuse, run_dlmuse_pipeline diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index 026353a0..a58ef372 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -6,21 +6,11 @@ """ import argparse -import os -import shutil -import threading - -from .dlmuse_pipeline import run_pipeline -from .utils import ( - collect_T1, - merge_bids_output_data, - merge_output_data, - remove_subfolders, - split_data, -) + +from .dlmuse_pipeline import run_dlmuse_pipeline # VERSION = pkg_resources.require("NiChart_DLMUSE")[0].version -VERSION = "1.0.7" +VERSION = "1.0.9" def main() -> None: @@ -142,94 +132,25 @@ def main() -> None: device = args.device dlicv_extra_args = args.dlicv_args dlmuse_extra_args = args.dlmuse_args + clear_cache = args.clear_cache + bids = args.bids + cores = args.cores print() print("Arguments:") print(args) print() - if not os.path.isdir(out_dir): - print(f"Can't find {out_dir}, creating it...") - # os.system(f"mkdir {out_dir}") - os.mkdir(out_dir) - - elif len(os.listdir(out_dir)) != 0: - print(f"Emptying output folder: {out_dir}...") - for root, dirs, files in os.walk(out_dir): - for f in files: - os.unlink(os.path.join(root, f)) - for d in dirs: - shutil.rmtree(os.path.join(root, d)) - - if args.clear_cache: - os.system("DLICV -i ./dummy -o ./dummy --clear_cache") - os.system("DLMUSE -i ./dummy -o ./dummy --clear_cache") - - working_dir = os.path.join(os.path.abspath(out_dir)) - - # Run pipeline - if args.bids is True: - if int(args.cores) > 1: - collect_T1(in_dir, out_dir) - - no_threads = int(args.cores) - subfolders = split_data("raw_temp_T1", no_threads) - threads = [] - for i in range(len(subfolders)): - curr_out_dir = out_dir + f"/split_{i}" - curr_thread = threading.Thread( - target=run_pipeline, - args=( - subfolders[i], - curr_out_dir, - device, - dlmuse_extra_args, - dlicv_extra_args, - i, - ), - ) - curr_thread.start() - threads.append(curr_thread) - - for t in threads: - t.join() - - merge_bids_output_data(working_dir) - remove_subfolders("raw_temp_T1") - remove_subfolders(out_dir) - else: # No core parallelization - run_pipeline(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) - - else: # Non-BIDS - if int(args.cores) > 1: - no_threads = int(args.cores) - subfolders = split_data(in_dir, no_threads) - - threads = [] - for i in range(len(subfolders)): - curr_out_dir = out_dir + f"/split_{i}" - curr_thread = threading.Thread( - target=run_pipeline, - args=( - subfolders[i], - curr_out_dir, - device, - dlmuse_extra_args, - dlicv_extra_args, - i, - ), - ) - curr_thread.start() - threads.append(curr_thread) - - for t in threads: - t.join() - - merge_output_data(out_dir) - remove_subfolders(in_dir) - remove_subfolders(out_dir) - else: # No core parallelization - run_pipeline(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) + run_dlmuse_pipeline( + in_dir, + out_dir, + device, + dlicv_extra_args, + dlmuse_extra_args, + clear_cache, + bids, + cores, + ) if __name__ == "__main__": diff --git a/NiChart_DLMUSE/dlmuse_pipeline.py b/NiChart_DLMUSE/dlmuse_pipeline.py index 8268fd79..5a89eaf6 100644 --- a/NiChart_DLMUSE/dlmuse_pipeline.py +++ b/NiChart_DLMUSE/dlmuse_pipeline.py @@ -1,5 +1,8 @@ import logging import os +import shutil +import threading +from typing import Any import pkg_resources # type: ignore @@ -8,7 +11,14 @@ from .RelabelROI import apply_relabel_rois from .ReorientImage import apply_reorient_img, apply_reorient_to_init from .SegmentImage import run_dlicv, run_dlmuse -from .utils import make_img_list +from .utils import ( + collect_T1, + make_img_list, + merge_bids_output_data, + merge_output_data, + remove_subfolders, + split_data, +) # Config vars SUFF_LPS = "_LPS.nii.gz" @@ -35,17 +45,131 @@ logging.basicConfig(filename="pipeline.log", encoding="utf-8", level=logging.DEBUG) -def run_pipeline( +def run_dlmuse_pipeline( + in_dir: str, + out_dir: str, + device: str, + dlicv_extra_args: str, + dlmuse_extra_args: str, + clear_cache: bool, + bids: bool, + cores: str, +) -> None: + """ + NiChart pipeline + + :param in_dir: The input directory + :type in_dir: str + :param out_dir: The output directory + :type out_dir: str + :type device: cpu/cuda/mps + :param device: str + :param dlicv_extra_args: Extra arguments for DLICV API + :type dlicv_extra_args: str + :param dlmuse_extra_args: Extra arguments for DLMUSE API + :type dlmuse_extra_args: str + :param clear_cache: True if cache should be cleared + :type clear_cache: bool + :param bids: True if your input is a bids type folder + :type bids: bool + :param cores: The number of cores(default is 4) + :type cores: str + + :rtype: None + """ + if not os.path.isdir(out_dir): + print(f"Can't find {out_dir}, creating it...") + # os.system(f"mkdir {out_dir}") + os.mkdir(out_dir) + + elif len(os.listdir(out_dir)) != 0: + print(f"Emptying output folder: {out_dir}...") + for root, dirs, files in os.walk(out_dir): + for f in files: + os.unlink(os.path.join(root, f)) + for d in dirs: + shutil.rmtree(os.path.join(root, d)) + if clear_cache: + os.system("DLICV -i ./dummy -o ./dummy --clear_cache") + os.system("DLMUSE -i ./dummy -o ./dummy --clear_cache") + + working_dir = os.path.join(os.path.abspath(out_dir)) + + if bids is True: + if int(cores) > 1: + collect_T1(in_dir, out_dir) + + no_threads = int(cores) + subfolders = split_data("raw_temp_T1", no_threads) + threads = [] + for i in range(len(subfolders)): + curr_out_dir = out_dir + f"/split_{i}" + curr_thread = threading.Thread( + target=run_thread, + args=( + subfolders[i], + curr_out_dir, + device, + dlmuse_extra_args, + dlicv_extra_args, + i, + ), + ) + curr_thread.start() + threads.append(curr_thread) + + for t in threads: + t.join() + + merge_bids_output_data(working_dir) + remove_subfolders("raw_temp_T1") + remove_subfolders(out_dir) + else: # No core parallelization + run_thread(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) + + else: # Non-BIDS + if int(cores) > 1: + no_threads = int(cores) + subfolders = split_data(in_dir, no_threads) + + threads = [] + for i in range(len(subfolders)): + curr_out_dir = out_dir + f"/split_{i}" + curr_thread = threading.Thread( + target=run_thread, + args=( + subfolders[i], + curr_out_dir, + device, + dlmuse_extra_args, + dlicv_extra_args, + i, + ), + ) + curr_thread.start() + threads.append(curr_thread) + + for t in threads: + t.join() + + merge_output_data(out_dir) + remove_subfolders(in_dir) + remove_subfolders(out_dir) + else: # No core parallelization + run_thread(in_dir, out_dir, device, dlmuse_extra_args, dlicv_extra_args) + + +def run_thread( in_data: str, out_dir: str, device: str, - dlmuse_extra_args: str = '', - dlicv_extra_args: str = '', + dlmuse_extra_args: str = "", + dlicv_extra_args: str = "", sub_fldr: int = 1, - progress_bar = None, + progress_bar: Any = None, ) -> None: """ - NiChart pipeline + Run a thread of the pipeline :param in_data: the input directory :type in_data: str diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index f4c4e310..89c2a344 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -65,7 +65,9 @@ def remove_common_suffix(list_files: list) -> list: bnames = list_files if len(list_files) == 1: - if list_files[0].endswith('_T1'): # If there is a single image with suffix _T1, remove it + if list_files[0].endswith( + "_T1" + ): # If there is a single image with suffix _T1, remove it bnames = [x[0:-3] for x in bnames] return bnames @@ -347,7 +349,7 @@ def remove_subfolders(in_dir: str) -> None: def merge_output_data(in_dir: str) -> None: """ - Takes all the results from the temp_working_fir and moves them into + Takes all the results from the temp_working_dir and moves them into the output folder :param in_dir: the input directory diff --git a/requirements.txt b/requirements.txt index 79414b1a..4d118458 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Main pipeline -torch==2.2.1 +torch==2.3.1 DLICV DLMUSE nibabel>=5.2 @@ -12,3 +12,6 @@ huggingface_hub argparse pathlib pre-commit + +# testing +pandas diff --git a/setup.py b/setup.py index 9698ec15..5bb55bc4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ url="https://github.com/CBICA/NiChart_DLMUSE", python_requires=">=3.9", install_requires=[ - "torch<=2.2.1", + "torch<=2.3.1", "DLICV", "DLMUSE", "huggingface_hub", diff --git a/tests/test_utils.py b/tests/test_utils.py index f61e764e..e364ec25 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -58,7 +58,19 @@ def testing_make_img_list() -> None: def testing_get_bids_prefix() -> None: - pass + temp_file = "test-1234" + assert get_bids_prefix(temp_file) == "test" + temp_file = "test-1" + assert get_bids_prefix(temp_file) == "test" + temp_file = "test" + assert get_bids_prefix(temp_file) == "test" + + temp_folder = "test_1234" + assert get_bids_prefix(temp_folder, True) == "test" + temp_folder = "test_1" + assert get_bids_prefix(temp_folder, True) == "test" + temp_folder = "test" + assert get_bids_prefix(temp_folder, True) == "test" def testing_collect_T1() -> None: From 30e54f3ae4127319abbcd2cf7f079c1ef0095d32 Mon Sep 17 00:00:00 2001 From: spirosmaggioros Date: Fri, 10 Jan 2025 00:18:13 +0000 Subject: [PATCH 2/7] Update macos tests to run with macos latest --- .github/workflows/macos_tests.yml | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/macos_tests.yml b/.github/workflows/macos_tests.yml index c8df2737..726c9144 100644 --- a/.github/workflows/macos_tests.yml +++ b/.github/workflows/macos_tests.yml @@ -4,25 +4,25 @@ name: macos build on: [push, pull_request, workflow_dispatch] jobs: - build: - runs-on: ["macos-13"] + build: + runs-on: ["macos-latest"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install spare scores - run: | - python -m pip cache purge - pip install -r requirements.txt - pip install setuptools twine wheel - python -m pip install . - - name: Run unit tests - run: | - cd tests/ && pytest --cov=../ --cov-report=xml - - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v4.0.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: CBICA/NiChart_DLMUSE + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install spare scores + run: | + python -m pip cache purge + pip install -r requirements.txt + pip install setuptools twine wheel + python -m pip install . + - name: Run unit tests + run: | + cd tests/ && pytest --cov=../ --cov-report=xml + - name: Upload Coverage to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: CBICA/NiChart_DLMUSE From ff253a76459181119ce6443dbc0efafb3ee92bbe Mon Sep 17 00:00:00 2001 From: spirosmaggioros Date: Fri, 10 Jan 2025 16:13:40 +0000 Subject: [PATCH 3/7] Changed pipeline function's name to run_ndlmuse_pipeline to avoid conflicts with dlmuse's pipeline --- NiChart_DLMUSE/__main__.py | 4 ++-- NiChart_DLMUSE/dlmuse_pipeline.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index a58ef372..fec91873 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -7,7 +7,7 @@ import argparse -from .dlmuse_pipeline import run_dlmuse_pipeline +from .dlmuse_pipeline import run_ndlmuse_pipeline # VERSION = pkg_resources.require("NiChart_DLMUSE")[0].version VERSION = "1.0.9" @@ -141,7 +141,7 @@ def main() -> None: print(args) print() - run_dlmuse_pipeline( + run_ndlmuse_pipeline( in_dir, out_dir, device, diff --git a/NiChart_DLMUSE/dlmuse_pipeline.py b/NiChart_DLMUSE/dlmuse_pipeline.py index 5a89eaf6..29b5e82a 100644 --- a/NiChart_DLMUSE/dlmuse_pipeline.py +++ b/NiChart_DLMUSE/dlmuse_pipeline.py @@ -45,7 +45,7 @@ logging.basicConfig(filename="pipeline.log", encoding="utf-8", level=logging.DEBUG) -def run_dlmuse_pipeline( +def run_ndlmuse_pipeline( in_dir: str, out_dir: str, device: str, From 24189284b7e1266e3eac158022def873e3a249ef Mon Sep 17 00:00:00 2001 From: spirosmaggioros Date: Fri, 10 Jan 2025 16:21:16 +0000 Subject: [PATCH 4/7] Correct function name in init --- NiChart_DLMUSE/__init__.py | 2 +- NiChart_DLMUSE/__main__.py | 2 +- NiChart_DLMUSE/dlmuse_pipeline.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/NiChart_DLMUSE/__init__.py b/NiChart_DLMUSE/__init__.py index 36e71672..1d89acb6 100644 --- a/NiChart_DLMUSE/__init__.py +++ b/NiChart_DLMUSE/__init__.py @@ -1 +1 @@ -from .dlmuse_pipeline import run_dlicv, run_dlmuse, run_dlmuse_pipeline +from .dlmuse_pipeline import run_dlicv, run_dlmuse, run_ndlmuse_pipeline diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index fec91873..cebec84e 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -7,7 +7,7 @@ import argparse -from .dlmuse_pipeline import run_ndlmuse_pipeline +from NiChart_DLMUSE.dlmuse_pipeline import run_ndlmuse_pipeline # VERSION = pkg_resources.require("NiChart_DLMUSE")[0].version VERSION = "1.0.9" diff --git a/NiChart_DLMUSE/dlmuse_pipeline.py b/NiChart_DLMUSE/dlmuse_pipeline.py index 29b5e82a..82183abf 100644 --- a/NiChart_DLMUSE/dlmuse_pipeline.py +++ b/NiChart_DLMUSE/dlmuse_pipeline.py @@ -6,12 +6,12 @@ import pkg_resources # type: ignore -from .CalcROIVol import apply_create_roi_csv, combine_roi_csv -from .MaskImage import apply_combine_masks, apply_mask_img -from .RelabelROI import apply_relabel_rois -from .ReorientImage import apply_reorient_img, apply_reorient_to_init -from .SegmentImage import run_dlicv, run_dlmuse -from .utils import ( +from NiChart_DLMUSE.CalcROIVol import apply_create_roi_csv, combine_roi_csv +from NiChart_DLMUSE.MaskImage import apply_combine_masks, apply_mask_img +from NiChart_DLMUSE.RelabelROI import apply_relabel_rois +from NiChart_DLMUSE.ReorientImage import apply_reorient_img, apply_reorient_to_init +from NiChart_DLMUSE.SegmentImage import run_dlicv, run_dlmuse +from NiChart_DLMUSE.utils import ( collect_T1, make_img_list, merge_bids_output_data, From 5d746bb5ba4d744114c80557aeb2a90c97557d60 Mon Sep 17 00:00:00 2001 From: spirosmaggioros Date: Fri, 10 Jan 2025 17:08:48 +0000 Subject: [PATCH 5/7] Make device parameter work like DLMUSE and DLICV --- NiChart_DLMUSE/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index cebec84e..1844e790 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -64,8 +64,7 @@ def main() -> None: # DEVICE argument parser.add_argument( - "-d", - "--device", + "-device", type=str, help="Device (cpu, cuda, or mps)", default=None, From c63f85fe43abe35ab249a1d14f3281c0ebdfcac7 Mon Sep 17 00:00:00 2001 From: spirosmaggioros Date: Fri, 10 Jan 2025 22:26:59 +0000 Subject: [PATCH 6/7] Reformat code --- NiChart_DLMUSE/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index 1844e790..87e5f31b 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -7,10 +7,11 @@ import argparse +import pkg_resources # type: ignore + from NiChart_DLMUSE.dlmuse_pipeline import run_ndlmuse_pipeline -# VERSION = pkg_resources.require("NiChart_DLMUSE")[0].version -VERSION = "1.0.9" +VERSION = pkg_resources.require("NiChart_DLMUSE")[0].version def main() -> None: From 9b35c0122d32b7d4281811a272ae15826c51eb0f Mon Sep 17 00:00:00 2001 From: spirosmaggioros Date: Sat, 11 Jan 2025 00:17:34 +0000 Subject: [PATCH 7/7] Remove unused code --- NiChart_DLMUSE/__main__.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index 87e5f31b..6b8a294b 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -127,29 +127,20 @@ def main() -> None: args = parser.parse_args() - in_dir = args.in_dir - out_dir = args.out_dir - device = args.device - dlicv_extra_args = args.dlicv_args - dlmuse_extra_args = args.dlmuse_args - clear_cache = args.clear_cache - bids = args.bids - cores = args.cores - print() print("Arguments:") print(args) print() run_ndlmuse_pipeline( - in_dir, - out_dir, - device, - dlicv_extra_args, - dlmuse_extra_args, - clear_cache, - bids, - cores, + args.in_dir, + args.out_dir, + args.device, + args.dlicv_extra_args, + args.dlmuse_extra_args, + args.clear_cache, + args.bids, + args.cores, )