diff --git a/examples/fuse_examples/fuse_examples_utils.py b/examples/fuse_examples/fuse_examples_utils.py new file mode 100644 index 000000000..940b9e3e1 --- /dev/null +++ b/examples/fuse_examples/fuse_examples_utils.py @@ -0,0 +1,15 @@ +import os +def ask_user(yes_no_question): + res = '' + while res not in ['y', 'n']: + res = input(f'{yes_no_question}? [y/n]') + return res =='y' + +def get_fuse_examples_user_dir(): + if 'USER_HOME_PATH' in os.environ: + fuse_examples_dir = os.path.join(os.environ['USER_HOME_PATH'],'fuse_examples') + else: + fuse_examples_dir = './fuse_examples' + if not os.path.exists(fuse_examples_dir): + os.mkdir(fuse_examples_dir) + return fuse_examples_dir diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/__init__.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/__init__.py new file mode 100644 index 000000000..d14d0073b --- /dev/null +++ b/examples/fuse_examples/imaging/classification/duke_breast_cancer/__init__.py @@ -0,0 +1,12 @@ +import os + +from fuse_examples.fuse_examples_utils import get_fuse_examples_user_dir + + +def get_duke_user_dir(): + return os.path.join(get_fuse_examples_user_dir(), 'duke') + +def get_duke_radiomics_user_dir(): + return os.path.join(get_fuse_examples_user_dir(), ' duke_radiomics') + + diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset.py deleted file mode 100644 index cd858403d..000000000 --- a/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset.py +++ /dev/null @@ -1,195 +0,0 @@ -import pandas as pd -from functools import partial -from fuse.data.augmentor.augmentor_default import AugmentorDefault -from fuse.data.augmentor.augmentor_toolbox import unsqueeze_2d_to_3d, aug_op_affine, squeeze_3d_to_2d, \ - rotation_in_3d -from fuse.data.dataset.dataset_generator import DatasetGenerator - -from fuse.utils.rand.param_sampler import Uniform, RandInt, RandBool -from fuse.data.visualizer.visualizer_default_3d import Fuse3DVisualizerDefault - -from fuse.data.processor.processor_dicom_mri import DicomMRIProcessor - -from fuse_examples.imaging.classification.prostate_x.patient_data_source import ProstateXDataSourcePatient - - -from fuse_examples.imaging.classification.duke_breast_cancer.post_processor import post_processing -from fuse_examples.imaging.classification.duke_breast_cancer.processor import PatchProcessor - - -def process_mri_series(metadata_path: str): - - seq_to_use = ['DCE_mix_ph1', - 'DCE_mix_ph2', - 'DCE_mix_ph3', - 'DCE_mix_ph4', - 'DCE_mix', - 'DCE_mix_ph', - 'MASK'] - subseq_to_use = ['DCE_mix_ph2', 'MASK'] - - l_seq = pd.read_csv(metadata_path) - seq_to_use_full = list(l_seq['Series Description'].value_counts().keys()) - - SER_INX_TO_USE = {} - SER_INX_TO_USE['all'] = {'DCE_mix': [1], 'MASK': [0]} - SER_INX_TO_USE['Breast_MRI_120'] = {'DCE_mix': [2], 'MASK': [0]} - SER_INX_TO_USE['Breast_MRI_596'] = {'DCE_mix': [2], 'MASK': [0]} - exp_patients = ['Breast_MRI_120','Breast_MRI_596'] - opt_seq = [ - '1st','1ax','1Ax','1/ax', - '2nd','2ax','2Ax','2/ax', - '3rd','3ax','3Ax', '3/ax', - '4th','4ax','4Ax','4/ax', - ] - my_keys = ['DCE_mix_ph1'] * 4 + ['DCE_mix_ph2'] * 4 + ['DCE_mix_ph3'] * 4 + ['DCE_mix_ph4'] * 4 - seq_to_use_full_slash = [s.replace('ax','/ax') for s in seq_to_use_full] - seq_to_use_full_slash = [s.replace('Ax', '/Ax') for s in seq_to_use_full_slash] - - seq_to_use_dict = {} - for opt_seq_tmp, my_key in zip(opt_seq, my_keys): - tt = [s for s in seq_to_use_full if opt_seq_tmp in s]+[s for s in seq_to_use_full_slash if opt_seq_tmp in s] - - for tmp in tt: - seq_to_use_dict[tmp] = my_key - seq_to_use_dict['ax dyn'] = 'DCE_mix_ph' - return seq_to_use_dict,SER_INX_TO_USE,exp_patients,seq_to_use,subseq_to_use - - - -def duke_breast_cancer_dataset(paths,train_common_params,lgr): - # ============================================================================== - # Data - # ============================================================================== - #### Train Data - - lgr.info(f'Train Data:', {'attrs': 'bold'}) - - ## Create data source: - DATABASE_REVISION = train_common_params['partition_version'] - lgr.info(f'database_revision={DATABASE_REVISION}', {'color': 'magenta'}) - - # create data source - train_data_source = ProstateXDataSourcePatient(paths['data_dir'], 'train', - db_ver=train_common_params['partition_version'], - db_name=train_common_params['db_name'], - fold_no=train_common_params['fold_no']) - - ## Create data processors: - image_processing_args = { - 'patch_xy': 100, - 'patch_z': 9, - } - - ## Create data processor - ######################################################################################### - seq_dict, SER_INX_TO_USE, exp_patients,seq_to_use,subseq_to_use = \ - process_mri_series(paths['metadata_path']) - mri_vol_processor = DicomMRIProcessor(seq_dict=seq_dict, - seq_to_use=seq_to_use, - subseq_to_use=subseq_to_use, - ser_inx_to_use=SER_INX_TO_USE, - exp_patients=exp_patients, - reference_inx=0, - use_order_indicator=False) - - generate_processor = PatchProcessor( - vol_processor=mri_vol_processor, - path_to_db=paths['data_dir'], - data_path=paths['data_path'], - ktrans_data_path='', - db_name=train_common_params['db_name'], - db_version=train_common_params['partition_version'], - fold_no=train_common_params['fold_no'], - lsn_shape=(image_processing_args['patch_z'], - image_processing_args['patch_xy'], - image_processing_args['patch_xy']), - ) - - train_post_processor = partial(post_processing, label=train_common_params['classification_task']) - - # data augmentation (optional) - num_channels = train_common_params['backbone_model_dict']['input_channels_num'] + 1 - slice_num = image_processing_args['patch_z'] - - _no_aug = [list(range(0, slice_num))] - aug_pipeline = [ - [ - ('data.input',), - rotation_in_3d, - {'z_rot': Uniform(-5.0, 5.0), 'y_rot': Uniform(-5.0, 5.0), 'x_rot': Uniform(-5.0, 5.0)}, - {'apply': RandBool(0.5)} - ], - [ - ('data.input',), - squeeze_3d_to_2d, - {'axis_squeeze': 'z'}, - {} - ], - [ - ('data.input',), - aug_op_affine, - {'rotate': Uniform(0, 360.0), - 'translate': (RandInt(-4, 4), RandInt(-4, 4)), - 'flip': (RandBool(0.5), RandBool(0.5)), - 'scale': Uniform(0.9, 1.1), - }, - {'apply': RandBool(0.5)} - ], - - [ - ('data.input',), - unsqueeze_2d_to_3d, - {'channels': num_channels, 'axis_squeeze': 'z'}, - {} - ], - ] - augmentor = AugmentorDefault(augmentation_pipeline=aug_pipeline) - - visualizer = Fuse3DVisualizerDefault(image_name='data.input', label_name='data.isLargeTumorSize') - # Create dataset - train_dataset = DatasetGenerator(cache_dest=paths['cache_dir'], - data_source=train_data_source, - processor=generate_processor, - post_processing_func=train_post_processor, - augmentor=augmentor, - statistic_keys=['data.ground_truth'], - visualizer=visualizer, - ) - - lgr.info(f'- Load and cache data:') - train_dataset.create() - - train_dataset.filter('data.filter', [True]) - lgr.info(f'- Load and cache data: Done') - - #### Validation data - lgr.info(f'Validation Data:', {'attrs': 'bold'}) - - ## Create data source - validation_data_source = ProstateXDataSourcePatient(paths['data_dir'], 'validation', - db_ver=DATABASE_REVISION, - db_name=train_common_params['db_name'], - fold_no=train_common_params['fold_no']) - - # post processor - validation_post_processor = partial(post_processing, label=train_common_params['classification_task']) - - ## Create dataset - validation_dataset = DatasetGenerator(cache_dest=paths['cache_dir'], - data_source=validation_data_source, - processor=generate_processor, - post_processing_func=validation_post_processor, - augmentor=None, - statistic_keys=['data.ground_truth'], - visualizer=None - ) - - lgr.info(f'- Load and cache data:') - - validation_dataset.create(num_workers=0) - lgr.info(f'Data - task caching and filtering:', {'attrs': 'bold'}) - - validation_dataset.filter('data.filter', [True]) - - return train_dataset,validation_dataset \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset_DUKE_folds_ver10012022Recurrence_seed1.pickle b/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset_DUKE_folds_ver10012022Recurrence_seed1.pickle deleted file mode 100644 index 68ad4683a..000000000 Binary files a/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset_DUKE_folds_ver10012022Recurrence_seed1.pickle and /dev/null differ diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset_DUKE_folds_ver11102021TumorSize_seed1.pickle b/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset_DUKE_folds_ver11102021TumorSize_seed1.pickle deleted file mode 100755 index e7f499d91..000000000 Binary files a/examples/fuse_examples/imaging/classification/duke_breast_cancer/dataset_DUKE_folds_ver11102021TumorSize_seed1.pickle and /dev/null differ diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/__init__.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/__init__.py new file mode 100644 index 000000000..23b94ba33 --- /dev/null +++ b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/__init__.py @@ -0,0 +1,61 @@ +import gzip +import os +import pickle + +import pandas as pd + + +def save_object(obj, filename): + open_func = gzip.open if filename.endswith(".gz") else open + filename_tmp = filename+ ".del" + if os.path.exists(filename_tmp): + os.remove(filename_tmp) + with open_func(filename_tmp, 'wb') as output: + pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL) + + os.rename(filename_tmp, filename) + return filename + + +def load_object(filename): + open_func = gzip.open if filename.endswith(".gz") else open + + with open_func(filename, 'rb') as myinput: + try: + res = pickle.load(myinput) + except RuntimeError as e: + print("Failed to read", filename) + raise e + return res + + +DUKE_PROCESSED_FILE_DIR = '/projects/msieve_dev3/usr/common/duke_processed_files' + + +def get_duke_annotations_from_tal_df(): + annotations_path = os.path.join(DUKE_PROCESSED_FILE_DIR, 'dataset_DUKE_folds_ver11102021TumorSize_seed1.pickle') + with open(annotations_path, 'rb') as infile: + fold_annotations_dict = pickle.load(infile) + annotations_df = pd.concat( + [fold_annotations_dict[f'data_fold{fold}'] for fold in range(len(fold_annotations_dict))]) + return annotations_df + + +def get_col_mapping(): + return {'MRI Findings:Skin/Nipple Invovlement': 'Skin Invovlement', + 'US features:Tumor Size (cm)': 'Tumor Size US', + 'Mammography Characteristics:Tumor Size (cm)': 'Tumor Size MG', + 'MRI Technical Information:FOV Computed (Field of View) in cm': 'Field of View', + 'MRI Technical Information:Contrast Bolus Volume (mL)': 'Contrast Bolus Volume', + 'Demographics:Race and Ethnicity': 'Race', + 'MRI Technical Information:Manufacturer Model Name': 'Manufacturer', + 'MRI Technical Information:Slice Thickness': 'Slice Thickness', + 'MRI Findings:Multicentric/Multifocal': 'Multicentric', + 'Mammography Characteristics:Breast Density': 'Breast Density MG', + 'Tumor Characteristics:PR': 'PR', + 'Tumor Characteristics:HER2': 'HER2', + 'Tumor Characteristics:ER': 'ER', + 'Near Complete Response:Overall Near-complete Response: Stricter Definition': 'Near pCR Strict', + 'Tumor Characteristics:Staging(Tumor Size)# [T]': 'Staging Tumor Size', + 'Tumor Characteristics:Histologic type': 'Histologic type', + } diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_debug_main.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_debug_main.py new file mode 100644 index 000000000..ca14ba681 --- /dev/null +++ b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_debug_main.py @@ -0,0 +1,263 @@ +# import nibabel as nib + +import fuseimg.datasets.duke_label_type +from fuse_examples.imaging.classification import duke_breast_cancer +from fuse_examples.imaging.classification.duke_breast_cancer.debug import DUKE_PROCESSED_FILE_DIR, get_duke_annotations_from_tal_df, \ + get_col_mapping +from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe +from fuseimg.datasets import duke +from fuse.data.utils.sample import create_initial_sample +from fuse.data.ops import ops_cast +import torch +from deepdiff import DeepDiff +import SimpleITK as sitk +from fuse.utils import file_io +import time +import numpy as np +from tqdm import tqdm + +import getpass +import os + +from fuseimg.datasets.duke import get_duke_clinical_data_df + +import pandas as pd + + +def main(): + + timestr = time.strftime("%Y%m%d-%H%M%S") + if False: + sample_id = 'Breast_MRI_900' + + + static_pipeline = duke.Duke.static_pipeline(data_dir=os.environ["DUKE_DATA_PATH"], + select_series_func=duke.get_selected_series_index) + # k = 5 #ok + # k = 8 + # static_pipeline._ops_and_kwargs = static_pipeline._ops_and_kwargs[:k] + # static_pipeline._op_ids = static_pipeline._op_ids[:k] + + sample_dict = create_initial_sample(sample_id) + sample_dict = static_pipeline(sample_dict) + s = replace_stk_with_numpy(sample_dict.flatten()) + s_old = file_io.load_pickle(f'/tmp/ozery/s_old_stat.pkl') + # x_new = s['data.input.volume4D'] + # x_old = s_old['data.input.volume4D'][:,:,:,0] + x_new = s['data.input.patch_volume'] + x_old = s_old['data.input.patch_volume'] + print("ok", np.abs(x_new-x_old).max()) + # deep_diff_config = dict(ignore_nan_inequality=True) # , math_epsilon=0.0001) + # diff = DeepDiff(s_old, s, **deep_diff_config) + # if len(diff) > 0: + # print(diff.keys()) + + if True: + sample_ids = ['Breast_MRI_900'] + + # sample_ids = duke.get_samples_for_debug(data_dir=os.environ["DUKE_DATA_PATH"], n_pos=10, n_neg=10, + # label_type=duke.DukeLabelType.STAGING_TUMOR_SIZE) + duke_dataset = duke.Duke.dataset(data_dir=os.environ["DUKE_DATA_PATH"], label_type=fuseimg.datasets.duke_label_type.DukeLabelType.STAGING_TUMOR_SIZE, + cache_dir=None, num_workers=0, sample_ids=sample_ids) + print("finished defining dataset, starting run") + arr = [] + rows = [] + deep_diff_config = dict(ignore_nan_inequality=True) # , math_epsilon=0.0001) + for d in duke_dataset: + d2 = replace_tensors_with_numpy(d.flatten()) + row = (d['data.sample_id'], d['data.ground_truth']) + rows.append(row) + print("*******", row) + arr +=[d2] + output_file = f'/tmp/ozery/s{sample_ids[0]}.pkl' + d2_old = replace_tensors_with_numpy(load_pickle( output_file).flatten()) + diff = DeepDiff(d2_old, d2, **deep_diff_config) + if len(diff) > 0: + print(sample_ids[0], "has diff") + print("wrote", output_file) + break + print(len(arr)) + print(pd.DataFrame(rows, columns=['sample_id', 'gt'])) + print(arr[0].keys()) + + else: + sample_id = 'Breast_MRI_596' # 'Breast_MRI_120'#'Breast_MRI_900' #'Breast_MRI_127' + sample_ids = duke.Duke.sample_ids()[:5] + for sample_id in tqdm(sample_ids): + output_file = f'/user/ozery/output/{sample_id}_v0.pkl' + if os.path.exists(output_file): + continue + static_pipeline = duke.Duke.static_pipeline(data_dir=root_path, select_series_func=duke.get_selected_series_index) + + sample_dict = create_initial_sample(sample_id) + sample_dict = static_pipeline(sample_dict) + # print("ok", sample_dict.flatten().keys()) + # save_pickle_safe(sample_dict, output_file) + # print("saved", output_file) + +def replace_tensors_with_numpy(d): + d2 = {} + for k, v in d.items(): + + if isinstance(v, torch.Tensor): + v = ops_cast.Cast.to_numpy(v) + print(k, "tensor => numpy") + d2[k] = v + + return d2 +def derive_fuse2_folds_files(): + input_filename_pattern = os.path.join(DUKE_PROCESSED_FILE_DIR, "dataset_DUKE_folds_ver{name}_seed1.pickle") + output_path = f'/projects/msieve_dev3/usr/{getpass.getuser()}/fuse_examples/duke' + output_filename_pattern = os.path.join(output_path, "DUKE_folds_fuse2_{name}_seed1.pkl") + + for name in ['10012022Recurrence', '11102021TumorSize']: + input_filename = input_filename_pattern.format(name=name) + output_filename = output_filename_pattern.format(name=name) + dict_in = load_pickle(input_filename) + + dict_out = {fold: dict_in[f'data_fold{fold}']['Patient ID'].values.tolist() for fold in range(5)} + a = [len(v) for v in dict_out.values()] + print(sum(a), a) + save_pickle_safe(dict_out, output_filename) + print("wrote", output_filename) + + +def compare_sample_dicts(file1, file2): + d1 = load_pickle(file1).flatten() + d2 = load_pickle(file2).flatten() + set1 = set(d1.keys()) + set2 = set(d2.keys()) + if set1==set2: + print("same keys") + else: + print("set1-set2", set1-set2, "set2-set1", set2-set1) + for k in set1.intersection(set2): + v1 = d1[k] + v2 = d2[k] + print(k, type(v1), type(v1)== type(v2), type(v1[0]) if isinstance(v1, list) else '', type(v2[0]) if isinstance(v2, list) else '') + if k in ['data.input.volume4D', 'data.input.ref_volume', + 'data.input.volumes.DCE_mix', 'data.input.selected_volumes', 'data.input.selected_volumes_resampled']: + continue + if type(v1).__module__ == np.__name__: + assert (v1==v2).all() + elif isinstance(v1, list): + assert ((np.asarray(v1)==np.asarray(v2)).all()) + else: + assert (v1==v2) + print("OK") + + # map = {'data.sample_id': 'data.sample_id', 'data.input.sequence_ids': 'data.input.sequence_ids', + # 'data.input.sequence_path.DCE_mix': 'data.input.path.DCE_mix', + # 'data.input.sequence_volumes.DCE_mix': 'data.input.volumes.DCE_mix', + # 'data.input.sequence_selected_volume.DCE_mix': 'data.input.selected_volumes.DCE_mix', + # 'data.input.sequence_selected_path.DCE_mix': 'data.input.selected_path.DCE_mix', + # 'data.input.sequence_selected_volume_resampled.DCE_mix': 'data.input.selected_volumes_resampled.DCE_mix'} + # + # for s1, s2 in map.items(): + # print(s1, s2, s2 in d2.keys(), d1[s1]==d2[s2]) + + +def get_excluded_patients(do_print=True): + df = get_duke_annotations_from_tal_df() + all_sample_ids = duke.Duke.sample_ids() + print(df.shape, df.columns[0], len(all_sample_ids)) + excluded_sample_ids = sorted(list(set(all_sample_ids) - set(df.iloc[:,0].values))) + if do_print: + print("excluded:",len(excluded_sample_ids)) + print([s[11:] for s in excluded_sample_ids]) + return excluded_sample_ids + +def check_fuse_results(): + data_dir = duke_breast_cancer.get_duke_user_dir() + filename = os.path.join(data_dir,'model_dir/infer_dir/validation_set_infer.gz' ) + df = load_pickle(filename) + print(df.shape, df.columns) + excluded_sample_ids = get_excluded_patients(do_print=False) + excluded_in_df = set(df.id).intersection(set(excluded_sample_ids)) + print("ok") + +def visualize_image_from_cache(): + data_dir = duke_breast_cancer.get_duke_user_dir() + data_dir2 = os.path.join(data_dir, 'cache_dir/duke_cache_ver0/hash_b03e85135f7b2b2ad5aa02b372920317') + filename = os.path.join(data_dir2, 'out_sample_id@ffebf1d99a8358183f7b031c842c7c84.pkl.gz') + sample_dict = load_pickle(filename) + dynamic_pipeline = duke.Duke.dynamic_pipeline(include_patch_by_mask=True, include_patch_fixed=True, verbose=True) + sample_dict = dynamic_pipeline(sample_dict) + + print(type(sample_dict)) + +def replace_tensors_with_numpy(d): + d2 = {} + for k, v in d.items(): + + if isinstance(v, torch.Tensor): + v = ops_cast.Cast.to_numpy(v) + print(k, "tensor => numpy") + d2[k] = v + return d2 + +def replace_stk_with_numpy(d): + d2 = {} + for k, v in d.items(): + if isinstance(v, sitk.Image): + v = sitk.GetArrayFromImage(v) + print(k, "tensor => numpy") + d2[k] = v + return d2 + +def f(d): + from fuse.utils import file_io + file_io.save_pickle_safe(d, '/tmp/ozery/s_old.pkl') + from fuse.utils import file_io + s_old = file_io.load_pickle('/tmp/ozery/s_old.pkl') + s = None + + from deepdiff import DeepDiff + s_old = file_io.load_pickle('/tmp/ozery/s_old.pkl') + deep_diff_config = dict(ignore_nan_inequality=True) # , math_epsilon=0.0001) + diff = DeepDiff(s_old, s, **deep_diff_config) + if len(diff) > 0: + print(diff.keys()) + +def cmp_features(): + data_dir = '/projects/msieve2/Platform/BigMedilytics/Data/Duke-Breast-Cancer-MRI' + + map = get_col_mapping() + cols_new = list(map.keys()) + cols_old = [map[k] for k in cols_new] + df_new = get_duke_clinical_data_df(data_dir).set_index('Patient Information:Patient ID')[cols_new] + df_old = get_duke_annotations_from_tal_df().set_index('Patient ID DICOM')[cols_old] + df_new = df_new.loc[df_old.index] + for icol in range(len(map)): + v_new = df_new.iloc[:, icol] + v_old = df_old.iloc[:, icol] + + v_new0 = v_new[0] + v_old0 = v_old[0] + # print("**", cols_old[icol], v_new0, v_old0, type(v_new0), type(v_old0)) + is_same = (v_new == v_old) + if isinstance(v_new0, np.float64) and isinstance(v_old0, np.float64): + is_same |= np.isnan(v_new) & np.isnan(v_old) + if (is_same).all(): + print(cols_old[icol], 'OK') + else: + ix = np.where(~is_same)[0] + diff = (~is_same) + print("----", cols_old[icol], 'ERROR!!!', df_new.index[ix[0]], v_new[ix[0]], v_old[ix[0]], f"#diff={diff.sum()} / {diff.shape[0]}") + + + +if __name__ == "__main__": + baseline_output_file = '/user/ozery/output/baseline1.pkl' # '/tmp/f2.pkl' + output_file = '/user/ozery/output/f5.pkl' + main() + # get_excluded_patients(do_print=True) + + + # compare_sample_dicts('/user/ozery/output/Breast_MRI_900_v0.pkl','/user/ozery/output/Breast_MRI_900_20220531-232611.pkl') + # derive_fuse2_folds_files() + # check_fuse_results() + # visualize_image_from_cache() + + + diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_mri_sequence_stats_main.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_mri_sequence_stats_main.py new file mode 100644 index 000000000..76f338c81 --- /dev/null +++ b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_mri_sequence_stats_main.py @@ -0,0 +1,77 @@ +import fuse_examples.imaging.classification.duke_breast_cancer.debug +import fuse_examples.imaging.classification.duke_breast_cancer.debug.duke_debug_main +from fuseimg.datasets import duke +from fuseimg.data.ops import ops_mri +from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe + +from tqdm import tqdm +import os +import pandas as pd + + +def main(): + root_path = '/projects/msieve2/Platform/BigMedilytics/Data/Duke-Breast-Cancer-MRI/manifest-1607053360376/' + data_path = os.path.join(root_path, 'Duke-Breast-Cancer-MRI') + metadata_path = os.path.join(root_path, 'metadata.csv') + + series_desc_2_sequence_map = duke.get_series_desc_2_sequence_mapping(metadata_path) + sample_ids = duke.Duke.sample_ids() + + filename = '/user/ozery/output/duke_samples.pkl' + if not os.path.exists(filename): + samples_info = get_samples_info(sample_ids, data_path, series_desc_2_sequence_map) + save_pickle_safe(samples_info, filename) + else: + samples_info = load_pickle(filename) + + df, cols_2_group = get_samples_stats(samples_info) + df_g = get_group_statistics(df.reset_index(), cols_2_group) + df_g.to_csv('/user/ozery/output/duke_stats.csv', index=False) + + annotations_df = fuse_examples.imaging.classification.duke_breast_cancer.debug.get_duke_annotations_from_tal_df() + df2 = df.loc[annotations_df['Patient ID']] + df2_g = get_group_statistics(df2.reset_index(), cols_2_group) + df2_g.to_csv('/user/ozery/output/duke_stats2.csv', index=False) + + +def get_samples_stats(samples_info): + DCE_mix_ph1_4 = [f'DCE_mix_ph{i + 1}' for i in range(4)] + seq_ids = DCE_mix_ph1_4 + ['DCE_mix_ph', 'UNKNOWN'] + n_seq_ids = len(seq_ids) + seq_id_pos_map = dict(zip(seq_ids, range(n_seq_ids))) + + rows = [] + for sample_id, sample_info in samples_info.items(): + sample_encoding = ['0'] * n_seq_ids + for seq_id, seq_arr in sample_info.items(): + pos = seq_id_pos_map[seq_id] + sample_encoding[pos] = str(len(seq_arr)) + rows.append([sample_id] + sample_encoding) + + df = pd.DataFrame(rows, columns=['sample_id'] + seq_ids) + df['DCE_mix_ph1_4'] = 'C' + df['DCE_mix_ph1'] + df['DCE_mix_ph2'] + df['DCE_mix_ph3'] + df['DCE_mix_ph4'] + df = df.drop(DCE_mix_ph1_4, axis=1) + cols_2_group = ['DCE_mix_ph1_4', 'DCE_mix_ph', 'UNKNOWN'] + df = df[cols_2_group + ['sample_id']] + + return df.set_index('sample_id'), cols_2_group + + +def get_group_statistics(df, cols_2_group): + df2 = df.groupby(cols_2_group, as_index=False).count() + df2 = df2.rename({'sample_id': '# patients'}, axis=1) + df2 = df2.sort_values('# patients') + return df2 + + +def get_samples_info(sample_ids, data_path, series_desc_2_sequence_map): + samples_info = {} + for sample_id in tqdm(sample_ids): + sample_path = duke.get_sample_path(data_path, sample_id) + seq_2_info_map = ops_mri.extract_seq_2_info_map(sample_path, series_desc_2_sequence_map) + samples_info[sample_id] = seq_2_info_map + return samples_info + + +if __name__ == '__main__': + main() diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_regression_tests_main.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_regression_tests_main.py new file mode 100644 index 000000000..f25ab776e --- /dev/null +++ b/examples/fuse_examples/imaging/classification/duke_breast_cancer/debug/duke_regression_tests_main.py @@ -0,0 +1,117 @@ +import getpass +import os + +import torch +import numpy as np + +import fuse_examples.imaging.classification.duke_breast_cancer.debug +import fuseimg.datasets.duke_label_type + +from fuse.data.utils.sample import create_initial_sample +from fuse.data.ops import ops_cast +from fuseimg.datasets import duke +from fuse_examples.imaging.classification import duke_breast_cancer +from deepdiff import DeepDiff + + +def main(): + # check dynamic + + sample_ids_2_test = [f'Breast_MRI_{i:03d}' for i in list(range(900, 901)) + [120, 596]] + + cache_dir = 'cache_dir_v2' + print(f"============================ Comparing to {cache_dir} ========================") + test_duke(os.environ["DUKE_DATA_PATH"], cache_dir=os.path.join(duke_breast_cancer.get_duke_user_dir(), cache_dir), + sample_ids_2_test=sample_ids_2_test, check_dynamic=False, verbose=False) + + cache_dir = 'cache_dir_regression_test' + print(f"============================ Comparing to {cache_dir} ========================") + test_duke(os.environ["DUKE_DATA_PATH"], + cache_dir=os.path.join(duke_breast_cancer.get_duke_user_dir(), cache_dir), + sample_ids_2_test=sample_ids_2_test, check_dynamic=True, verbose=False) + + + + +def test_duke(root_path, cache_dir, sample_ids_2_test, check_dynamic, verbose=False): + label_type = fuseimg.datasets.duke_label_type.DukeLabelType.STAGING_TUMOR_SIZE + + static_pipeline = duke.Duke.static_pipeline(data_dir=root_path, + select_series_func=duke.get_selected_series_index, + verbose=verbose) + dict_static = {} + + if check_dynamic: + dynamic_pipeline = duke.Duke.dynamic_pipeline(data_dir=root_path, label_type=label_type, verbose=verbose) + dict_dynamic = {} + for sample_id in sample_ids_2_test: + sample_dict = create_initial_sample(sample_id) + sample_dict = static_pipeline(sample_dict) + dict_static[sample_id] = sample_dict.flatten() + + if check_dynamic: + sample_dict = dynamic_pipeline(sample_dict) + dict_dynamic[sample_id] = replace_tensors_with_numpy(sample_dict.flatten()) + + to_check = [('static', dict_static)] + if check_dynamic: + to_check += [('dynamic', dict_dynamic)] + + for s, dict_obj in to_check: + print(f"--------------------- {sample_id}: test {s}") + pipeline_cache_file = os.path.join(cache_dir, f'{s}_pipeline_output.pkl.gz') + if os.path.exists(pipeline_cache_file): + ref_dict_obj = fuse_examples.imaging.classification.duke_breast_cancer.debug.load_object(pipeline_cache_file) + compare_dicts(dict_obj, ref_dict_obj) + else: + print("\tno previous results to compare to - writing current results for future tests") + fuse_examples.imaging.classification.duke_breast_cancer.debug.save_object(dict_obj, pipeline_cache_file) + print("wrote", pipeline_cache_file) + + +def replace_tensors_with_numpy(d): + d2 = {} + for k, v in d.items(): + + if isinstance(v, torch.Tensor): + if len(v.shape) > 2: + continue + v = ops_cast.Cast.to_numpy(v) + print(k, "tensor => numpy") + d2[k] = v + + return d2 + + +def compare_dicts(dict_obj, ref_dict_obj): + deep_diff_config = dict(ignore_nan_inequality=True) # , math_epsilon=0.0001) + for sample_id, sample_dict in dict_obj.items(): + # keys_to_compare = [s for s in sample_dict.keys() if s.startswith('data.input.patch_annotations.')] + # for k in keys_to_compare: + # v1 = sample_dict[k] + # v2 = ref_dict_obj[sample_id].get(k+'_T0') + # print(k, v1, v2, type(v1), type(v2)) + # if v2 is None: + # print('k does not exist') + # elif not isinstance(v1, tuple): + # print("---",k, np.abs(v1-v2)) + # assert np.abs(v1-v2) < 1e-5 + + diff = DeepDiff(ref_dict_obj[sample_id], sample_dict, **deep_diff_config) + minor_diffs = ['type_changes', 'dictionary_item_removed'] + + if len(diff) > 0: + if set(diff.keys()) - set(minor_diffs): + print("XXXXXXXXXXXXXXXXXXX", sample_id, "has major differences") + else: + print("\t\t", sample_id, "has minor differences") + print(diff.keys()) + for key in minor_diffs: + if key in diff: + print("----", key, diff[key]) + # print(f'{sample_id}\n{diff}') + print("Done") + + +if __name__ == '__main__': + main() diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/post_processor.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/post_processor.py deleted file mode 100644 index e72e5fe16..000000000 --- a/examples/fuse_examples/imaging/classification/duke_breast_cancer/post_processor.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -from typing import Dict -import torch -from fuse.utils.ndict import NDict -import numpy as np - - -def post_processing(batch_dict: NDict, label: str, is_concat_features_to_input: bool = False, - ) -> None: - """ - post_processing updates batch_dict on the post processing phase - This function : - 1. Defines the classification label base on label input und update the ground_truth field in batch_dict - 2. Extracts the relevant tubular parameters from add_data field in the batch_dict and - creates data.clinical_features - :param batch_dict: - :param label: label to use as classification label - :param is_concat_features_to_input - if True, concat the data.clinical_features to the input channels - :return: updated batch_dict - """ - - # select input channel - input_tensor = batch_dict['data.input'] - batch_dict['data.input'] = input_tensor[[0],:,:,:] - clinical_features = batch_dict['data.add_data'] - - - - # select label - mylabel = label - batch_dict['data.filter'] = False - - if mylabel == 'ispCR': - features_to_use = ['Skin Invovlement', 'Tumor Size US', 'Tumor Size MG', 'Field of View', 'Contrast Bolus Volume', - 'Race','Manufacturer','Slice Thickness'] - - ispCR = clinical_features['Near pCR Strict'] + 0 - if ispCR>2: - batch_dict['data.filter'] = True - return - elif ispCR==0 or ispCR==2: - ispCR = 0 - else: - ispCR = 1 - label_tensor = torch.tensor(ispCR, dtype = torch.int64) - batch_dict['data.ground_truth'] = label_tensor - - - if mylabel == 'Staging Tumor Size': - features_to_use = ['Skin Invovlement', 'Tumor Size US', 'Tumor Size MG', 'Field of View', 'Contrast Bolus Volume', - 'Race','Manufacturer','Slice Thickness'] - - TumorSize = clinical_features['Staging Tumor Size'] + 0 - if TumorSize>1: - TumorSize=1 - else: - TumorSize=0 - label_tensor = torch.tensor(TumorSize, dtype = torch.int64) - batch_dict['data.ground_truth'] = label_tensor - - if mylabel == 'Histology Type': - features_to_use = ['Skin Invovlement', 'Tumor Size US', 'Tumor Size MG', 'Field of View', 'Contrast Bolus Volume', - 'Race','Multicentric','Manufacturer','Slice Thickness'] - type = clinical_features['Histologic type'] + 0 - if type==1: - type=0 - elif type==10: - type=1 - else: - batch_dict['data.filter'] = True - - label_tensor = torch.tensor(type, dtype = torch.int64) - batch_dict['data.ground_truth'] = label_tensor - - - if mylabel == 'is High Tumor Grade Total': - features_to_use = ['Breast Density MG','PR','HER2','ER'] - grade = clinical_features['Tumor Grade Total'] + 0 - if grade>=7: - grade = 1 - else: - grade = 0 - - label_tensor = torch.tensor(grade, dtype = torch.int64) - batch_dict['data.ground_truth'] = label_tensor - - - - - # add clinical - - clinical_features_to_use = torch.tensor([float(clinical_features[feature]) for feature in features_to_use],dtype = torch.float32) - batch_dict['data.clinical_features'] = clinical_features_to_use - - if is_concat_features_to_input: - # select input channel - input_tensor = batch_dict['data.input'] - input_shape = input_tensor.shape - for feature in clinical_features_to_use: - input_tensor = torch.cat((input_tensor,feature.repeat(input_shape[1],input_shape[2],input_shape[3]).unsqueeze(0)),dim=0) - batch_dict['data.input'] = input_tensor \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/processor.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/processor.py deleted file mode 100644 index dea754456..000000000 --- a/examples/fuse_examples/imaging/classification/duke_breast_cancer/processor.py +++ /dev/null @@ -1,367 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -from typing import Tuple -import os -import SimpleITK as sitk -import numpy as np -import torch -import logging -import cv2 -from scipy.ndimage.morphology import binary_dilation -from fuse.data.processor.processor_base import ProcessorBase -from fuse.data.processor.processor_dicom_mri import DicomMRIProcessor - -from fuse_examples.imaging.classification.prostate_x.data_utils import ProstateXUtilsData - - -class PatchProcessor(ProcessorBase): - """ - This processor crops the lesion volume from within 4D MRI volume base on - lesion location as appears in the database. - :returns a sample that includes: - 'patient_num': patient id - 'lesion_num': one MRI volume may include more than one lesion - 'input': vol_tensor as extracted from MRI volume processor - 'input_lesion_mask': mask_tensor, - 'ggg': row['ggg']: in prostate - lesion grade - 'zone': row['zone']: zone in prostate - 'ClinSig': row['ClinSig']: Clinical significant ( 0 for benign and 3+3 lesions, 1 for rest) - """ - def __init__(self, - vol_processor: DicomMRIProcessor = DicomMRIProcessor(), - path_to_db: str = None, - data_path: str = None, - ktrans_data_path: str = None, - db_name: str = None, - db_version: str = None, - fold_no : int = None, - lsn_shape: Tuple[int, int, int] = (16, 120, 120), - lsn_spacing: Tuple[float, float, float] = (3, 0.5, 0.5), - longtd_inx: int = 0, - ): - """ - :param vol_processor - extracts 4D tensor from path to MRI dicoms - :param path_to_db: path to data pickle - :param data_path: path to directory in which dicom data is located - :param ktrans_data_path: path to directory of Ktrans seq (prostate x) - :param db_name: 'prostatex' for this example - :param fold_no: cross validation fold - :param lsn_shape: shape of volume to extract from full volume (pixels) - :param lsn_spacing: spacing of volume to extract from full volume (mm) - """ - - # store input parameters - self.vol_processor = vol_processor - self.path_to_db = path_to_db - self.data_path = data_path - self.ktrans_data_path = ktrans_data_path - self.lsn_shape = lsn_shape - self.lsn_spacing = lsn_spacing - self.db_name = db_name - self.db_ver = db_version - self.fold_no=fold_no - self.prostate_data_path = self.data_path - self.longtd_inx = longtd_inx - - - # ======================================================================== - def create_resample(self,vol_ref:sitk.sitkFloat32, interpolation: str, size:tuple, spacing: tuple): - """ - create_resample create resample operator - :param vol_ref: sitk vol to use as a ref - :param interpolation:['linear','nn','bspline'] - :param size: in pixels () - :param spacing: in mm () - :return: resample sitk operator - """ - - if interpolation == 'linear': - interpolator = sitk.sitkLinear - elif interpolation == 'nn': - interpolator = sitk.sitkNearestNeighbor - elif interpolation == 'bspline': - interpolator = sitk.sitkBSpline - - resample = sitk.ResampleImageFilter() - resample.SetReferenceImage(vol_ref) - resample.SetOutputSpacing(spacing) - resample.SetInterpolator(interpolator) - resample.SetSize(size) - return resample - - # ======================================================================== - def apply_resampling(self,img:sitk.sitkFloat32, mask:sitk.sitkFloat32, - spacing: Tuple[float,float,float] =(0.5, 0.5, 3), size: Tuple[int,int,int] =(160, 160, 32), - transform:sitk=None, interpolation:str='bspline', - label_interpolator:sitk=sitk.sitkLabelGaussian, - ): - - ref = img if img != [] else mask - size = [int(s) for s in size] - resample = self.create_resample(ref, interpolation, size=size, spacing=spacing) - - if ~(transform is None): - resample.SetTransform(transform) - img_r = resample.Execute(img) - - resample.SetInterpolator(label_interpolator) - mask_r = resample.Execute(mask) - - - return img_r, mask_r - - # ======================================================================== - def crop_lesion_vol(self,vol:sitk.sitkFloat32, position:Tuple[float,float,float], ref:sitk.sitkFloat32, size:Tuple[int,int,int]=(160, 160, 32), - spacing:Tuple[int,int,int]=(1, 1, 3), center_slice=None): - """ - crop_lesion_vol crop tensor around position - :param vol: vol to crop - :param position: point to crop around - :param ref: reference volume - :param size: size in pixels to crop - :param spacing: spacing to resample the col - :param center_slice: z coordinates of position - :return: cropped volume - """ - - def get_lesion_mask(position, ref): - mask = np.zeros_like(sitk.GetArrayViewFromImage(ref), dtype=np.uint8) - - coords = np.round(position[::-1]).astype(np.int) - mask[coords[0], coords[1], coords[2]] = 1 - mask = binary_dilation(mask, np.ones((3, 5, 5))) + 0 - mask_sitk = sitk.GetImageFromArray(mask) - mask_sitk.CopyInformation(ref) - - return mask_sitk - - mask = get_lesion_mask(position, ref) - - vol.SetOrigin((0,) * 3) - mask.SetOrigin((0,) * 3) - vol.SetDirection(np.eye(3).flatten()) - mask.SetDirection(np.eye(3).flatten()) - - ma_centroid = mask > 0.5 - label_analysis_filer = sitk.LabelShapeStatisticsImageFilter() - label_analysis_filer.Execute(ma_centroid) - centroid = label_analysis_filer.GetCentroid(1) - offset_correction = np.array(size) * np.array(spacing)/2 - corrected_centroid = np.array(centroid) - corrected_centroid[2] = center_slice * np.array(spacing[2]) - offset = corrected_centroid - np.array(offset_correction) - - translation = sitk.TranslationTransform(3, offset) - img, mask = self.apply_resampling(vol, mask, spacing=spacing, size=size, transform=translation) - - return img, mask - - - - # ======================================================================== - def crop_lesion_vol_mask_based(self,vol:sitk.sitkFloat32, position:tuple, ref:sitk.sitkFloat32, size:Tuple[int,int,int]=(160, 160, 32), - spacing:Tuple[int,int,int]=(1, 1, 3), mask_inx = -1,is_use_mask=True): - """ - crop_lesion_vol crop tensor around position - :param vol: vol to crop - :param position: point to crop around - :param ref: reference volume - :param size: size in pixels to crop - :param spacing: spacing to resample the col - :param center_slice: z coordinates of position - :param mask_inx: channel index in which mask is located default: last channel - :param is_use_mask: use mask to define crop bounding box - :return: cropped volume - """ - - margin = [20,20,0] - vol_np = sitk.GetArrayFromImage(vol) - if is_use_mask: - - mask = sitk.GetArrayFromImage(vol)[:,:,:,mask_inx] - mask_bool = np.zeros(mask.shape).astype(int) - mask_bool[mask>0.01]=1 - mask_final = sitk.GetImageFromArray(mask_bool) - mask_final.CopyInformation(ref) - - lsif = sitk.LabelShapeStatisticsImageFilter() - lsif.Execute(mask_final) - bounding_box = np.array(lsif.GetBoundingBox(1)) - vol_np[:, :, :, mask_inx] = mask_bool - else: - bounding_box = np.array([int(position[0]) - int(size[0] / 2), - int(position[1]) - int(size[1] / 2), - int(position[2]) - int(size[2] / 2), - size[0], - size[1], - size[2] - ]) - # in z use a fixed number of slices,based on position - bounding_box[-1] = size[2] - bounding_box[2] = int(position[2]) - int(size[2]/2) - - bounding_box_size = bounding_box[3:5][np.argmax(bounding_box[3:5])] - dshift = bounding_box[3:5] - bounding_box_size - dshift = np.append(dshift,0) - - ijk_min_bound = np.maximum(bounding_box[0:3]+dshift - margin,0) - ijk_max_bound = np.maximum(bounding_box[0:3]+dshift+[bounding_box_size,bounding_box_size,bounding_box[-1]] + margin,0) - - - - vol_np_cropped = vol_np[ijk_min_bound[2]:ijk_max_bound[2],ijk_min_bound[1]:ijk_max_bound[1],ijk_min_bound[0]:ijk_max_bound[0],:] - vol_np_resized = np.zeros((size[2],size[0],size[1],vol_np_cropped.shape[-1])) - for si in range(vol_np_cropped.shape[0]): - for ci in range(vol_np_cropped.shape[-1]): - vol_np_resized[si,:,:,ci] = cv2.resize(vol_np_cropped[si, :,:, ci], (size[0],size[1]), interpolation=cv2.INTER_AREA) - - img = sitk.GetImageFromArray(vol_np_resized) - mask = sitk.GetImageFromArray(vol_np_resized[:,:,:,mask_inx]) - - return img, mask - - - def get_zeros_vol(self,vol): - if vol.GetNumberOfComponentsPerPixel()>1: - ref_zeros_vol = sitk.VectorIndexSelectionCast(vol,0) - else: - ref_zeros_vol = vol - zeros_vol = np.zeros_like(sitk.GetArrayFromImage(ref_zeros_vol)) - zeros_vol = sitk.GetImageFromArray(zeros_vol) - zeros_vol.CopyInformation(ref_zeros_vol) - return zeros_vol - - def extract_mask_from_annotation(self,vol_ref,bbox_coords): - xstart = bbox_coords[0] - ystart = bbox_coords[1] - zstart = bbox_coords[2] - xsize = bbox_coords[3] - ysize = bbox_coords[4] - zsize = bbox_coords[5] - - mask = self.get_zeros_vol(vol_ref) - mask_np = sitk.GetArrayFromImage(mask) - mask_np[zstart:zstart+zsize,ystart:ystart+ysize,xstart:xstart+xsize] = 1.0 - return mask_np - - # ======================================================================== - def __call__(self, - sample_desc, - *args, **kwargs): - """ - Return list of samples (lesions) giving a patient level descriptor - :param sample_desc: (db_ver, set_type, patient_id) - :return: list of lesions, see TorchClassificationAlgo.create_lesion_sample() - """ - samples = [] - - # decode descriptor - patient_id = sample_desc - - # ======================================================================== - # get db - lesions - db_full = ProstateXUtilsData.get_dataset(self.path_to_db,'other',self.db_ver,self.db_name,self.fold_no) - db = ProstateXUtilsData.get_lesions_prostate_x(db_full) - - # ======================================================================== - # get patient - patient = db[db['Patient ID'] == patient_id] - # ======================================================================== - lgr = logging.getLogger('Fuse') - lgr.info(f'patient={patient_id}', {'color': 'magenta'}) - - - # ======================================================================== - # all seq paths for a certain patient - - - patient_directories = os.path.join(os.path.join(self.prostate_data_path, patient_id),patient['ser_name_T'+str(self.longtd_inx)].values[0][2:-2]) - images_path = os.path.join(self.prostate_data_path, patient_id, patient_directories) - - # ======================================================================== - # vol_4D is multichannel volume (z,x,y,chan(sequence)) - vol_4D,vol_ref = self.vol_processor((images_path,self.ktrans_data_path,patient_id)) - - # ======================================================================== - # each row contains one lesion, iterate over lesions - - for index, row in patient.iterrows(): - #read original position - pos_orig = np.fromstring(row['centroid_T'+str(self.longtd_inx)][1:-1], dtype=np.float32, sep=',') - - # transform to pixel coordinate in ref coords - pos_vol = np.array(vol_ref.TransformPhysicalPointToContinuousIndex(pos_orig.astype(np.float64))) - - vol_4d_tmp = sitk.GetArrayFromImage(vol_4D) - if sum(sum(sum(vol_4d_tmp[:,:,:,-1])))==0: - bbox_coords = np.fromstring(row['bbox_T' +str(self.longtd_inx)][1:-1],dtype = np.int32,sep=',') - mask = self.extract_mask_from_annotation(vol_ref,bbox_coords) - vol_4d_tmp[:,:,:,-1] = mask - vol_4d_new = sitk.GetImageFromArray(vol_4d_tmp) - vol_4D = vol_4d_new - - - - # crop lesion vol - resized to lsn_shape - vol_cropped_orig, mask_cropped_orig = self.crop_lesion_vol_mask_based( - vol_4D, pos_vol, vol_ref, - size=(2*self.lsn_shape[2], 2*self.lsn_shape[1], self.lsn_shape[0]), - spacing=(self.lsn_spacing[2], self.lsn_spacing[1], self.lsn_spacing[0]), mask_inx=-1,is_use_mask=False) - - # crop lesion vol - vol_cropped, mask_cropped = self.crop_lesion_vol_mask_based( - vol_4D, pos_vol,vol_ref , - size=(self.lsn_shape[2], self.lsn_shape[1], self.lsn_shape[0]), - spacing=(self.lsn_spacing[2], self.lsn_spacing[1], self.lsn_spacing[0]), mask_inx = -1,is_use_mask=True) - - - vol_cropped_tmp = sitk.GetArrayFromImage(vol_cropped) - vol_cropped_orig_tmp = sitk.GetArrayFromImage(vol_cropped_orig) - if len(vol_cropped_tmp.shape)<4: - # fix dimensions in case of one seq - vol_cropped_tmp = vol_cropped_tmp[:,:,:,np.newaxis] - vol = np.moveaxis(vol_cropped_tmp, 3, 0) - vol_cropped_orig_tmp = vol_cropped_orig_tmp[:, :, :, np.newaxis] - vol_orig = np.moveaxis(vol_cropped_orig_tmp, 3, 0) - else: - vol = np.moveaxis(sitk.GetArrayFromImage(vol_cropped), 3, 0) - vol_orig = np.moveaxis(sitk.GetArrayFromImage(vol_cropped_orig), 3, 0) - - if np.isnan(vol).any(): - input[np.isnan(input)] = 0 - - mask = sitk.GetArrayFromImage(mask_cropped) - vol_tensor = torch.from_numpy(vol).type(torch.FloatTensor) - mask_tensor = torch.from_numpy(mask).unsqueeze(0).type(torch.FloatTensor) - - mask_orig = sitk.GetArrayFromImage(mask_cropped_orig) - vol_tensor_orig = torch.from_numpy(vol_orig).type(torch.FloatTensor) - mask_tensor_orig = torch.from_numpy(mask_orig).unsqueeze(0).type(torch.FloatTensor) - - # sample - sample = { - 'patient_num': patient_id, - 'input': vol_tensor, - 'input_orig': vol_tensor_orig, - 'input_lesion_mask': mask_tensor, - 'add_data':row, - - } - - samples.append(sample) - - - - return samples diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/run_train_3dpatch.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/run_train_3dpatch.py deleted file mode 100644 index 091962efb..000000000 --- a/examples/fuse_examples/imaging/classification/duke_breast_cancer/run_train_3dpatch.py +++ /dev/null @@ -1,384 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -import logging -import os -import pathlib -from fuse.data.dataset.dataset_base import DatasetBase -import torch.nn.functional as F -import torch.optim as optim -from torch.utils.data.dataloader import DataLoader - -from fuse.eval.metrics.classification.metrics_classification_common import MetricROCCurve, MetricAUCROC -from fuse.eval.evaluator import EvaluatorDefault -from fuse.data.sampler.sampler_balanced_batch import SamplerBalancedBatch -from fuse.dl.losses.loss_default import LossDefault -from fuse.dl.managers.callbacks.callback_metric_statistics import MetricStatisticsCallback -from fuse.dl.managers.callbacks.callback_tensorboard import TensorboardCallback -from fuse.dl.managers.callbacks.callback_time_statistics import TimeStatisticsCallback -from fuse.dl.managers.manager_default import ManagerDefault - -import fuse.utils.gpu as GPU -from fuse.utils.utils_logger import fuse_logger_start - -from fuse.dl.models.heads import Head1DClassifier - -from fuse_examples.imaging.classification.prostate_x.backbone_3d_multichannel import Fuse_model_3d_multichannel,ResNet -from fuse_examples.imaging.classification.prostate_x.patient_data_source import ProstateXDataSourcePatient - - - -from fuse_examples.imaging.classification.duke_breast_cancer.dataset import duke_breast_cancer_dataset -from fuse_examples.imaging.classification.duke_breast_cancer.tasks import Task - - -########################################## -# Output Paths -# ########################################## - -# # TODO: path to save model -root_path = '.' - - -# TODO: path for duke data -# Download instructions can be found in README -root_data = 'Duke-Breast-Cancer-MRI/manifest-1607053360376' - -PATHS = {'force_reset_model_dir': False, - # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. - 'model_dir': os.path.join(root_path, 'duke/my_model/'), - 'cache_dir': os.path.join(root_path, 'duke/my_cache/'), - 'inference_dir': os.path.join(root_path, 'duke/my_model/inference/'), - 'eval_dir': os.path.join(root_path, 'duke/my_model/eval/'), - 'data_dir': pathlib.Path(__file__).parent.resolve(), - 'data_path' : os.path.join(root_data, 'Duke-Breast-Cancer-MRI'), - 'metadata_path': os.path.join(root_data, 'metadata.csv'), - } - - -################################# -# Train Template -################################# -########################################## -# Train Common Params -########################################## -# ============ -# Data -# ============ -TRAIN_COMMON_PARAMS = {} -TRAIN_COMMON_PARAMS['db_name'] = 'DUKE' -TRAIN_COMMON_PARAMS['partition_version'] = '11102021TumorSize' -TRAIN_COMMON_PARAMS['fold_no'] = 0 -TRAIN_COMMON_PARAMS['data.batch_size'] = 50 -TRAIN_COMMON_PARAMS['data.train_num_workers'] = 8 -TRAIN_COMMON_PARAMS['data.validation_num_workers'] = 8 - - -# =============== -# Manager - Train -# =============== -TRAIN_COMMON_PARAMS['manager.train_params'] = { - 'num_gpus': 1, - 'num_epochs': 5, - 'virtual_batch_size': 1, # number of batches in one virtual batch - 'start_saving_epochs': 120, # first epoch to start saving checkpoints from - 'gap_between_saving_epochs': 1, # number of epochs between saved checkpoint -} -TRAIN_COMMON_PARAMS['manager.best_epoch_source'] = [ - { - 'source': 'metrics.auc.macro_avg', # can be any key from losses or metrics dictionaries - 'optimization': 'max', # can be either min/max - 'on_equal_values': 'better', - # can be either better/worse - whether to consider best epoch when values are equal - }, -] -TRAIN_COMMON_PARAMS['manager.learning_rate'] = 1e-5 -TRAIN_COMMON_PARAMS['manager.weight_decay'] = 1e-3 -TRAIN_COMMON_PARAMS['manager.dropout'] = 0.5 -TRAIN_COMMON_PARAMS['manager.momentum'] = 0.9 -TRAIN_COMMON_PARAMS['manager.resume_checkpoint_filename'] = None - -TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] = 512 - -# in order to add relevant tabular feature uncomment: -# num_backbone_features_clinical, post_concat_inputs,post_concat_model -TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] = None#256 -TRAIN_COMMON_PARAMS['post_concat_inputs'] = None#[('data.clinical_features',9),] -TRAIN_COMMON_PARAMS['post_concat_model'] = None#(256,256) - -if TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] is None: - TRAIN_COMMON_PARAMS['num_backbone_features'] = TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] -else: - TRAIN_COMMON_PARAMS['num_backbone_features'] = \ - TRAIN_COMMON_PARAMS['num_backbone_features_imaging']+TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] - -# classification_task: -# supported tasks are: 'Staging Tumor Size','Histology Type','is High Tumor Grade Total','PCR' - -TRAIN_COMMON_PARAMS['classification_task'] = 'Staging Tumor Size' -TRAIN_COMMON_PARAMS['task'] = Task(TRAIN_COMMON_PARAMS['classification_task'], 0) -TRAIN_COMMON_PARAMS['class_num'] = TRAIN_COMMON_PARAMS['task'].num_classes() - -# backbone parameters -TRAIN_COMMON_PARAMS['backbone_model_dict'] = \ - {'input_channels_num': 1, - } - - - -def train_template(paths: dict, train_common_params: dict): - # ============================================================================== - # Logger - # ============================================================================== - fuse_logger_start(output_path=paths['model_dir'], console_verbose_level=logging.INFO, - list_of_source_files=[]) - lgr = logging.getLogger('Fuse') - lgr.info('Fuse Train', {'attrs': ['bold', 'underline']}) - - lgr.info(f'model_dir={os.path.abspath(paths["model_dir"])}', {'color': 'magenta'}) - lgr.info(f'cache_dir={os.path.abspath(paths["cache_dir"])}', {'color': 'magenta'}) - - #Data - train_dataset,validation_dataset = duke_breast_cancer_dataset(paths, train_common_params, lgr) - - ## Create dataloader - lgr.info(f'- Create sampler:') - - sampler = SamplerBalancedBatch(dataset=train_dataset, - balanced_class_name='data.ground_truth', - num_balanced_classes=train_common_params['class_num'], - batch_size=train_common_params['data.batch_size'], - balanced_class_weights= - [int(train_common_params['data.batch_size']/train_common_params['class_num'])] * train_common_params['class_num'], - use_dataset_cache=True) - - lgr.info(f'- Create sampler: Done') - - # ## Create dataloader - train_dataloader = DataLoader(dataset=train_dataset, - batch_sampler=sampler, - collate_fn=train_dataset.collate_fn, - num_workers=train_common_params['data.train_num_workers']) - lgr.info(f'Train Data: Done', {'attrs': 'bold'}) - - validation_dataloader = DataLoader(dataset=validation_dataset, - shuffle=False, - drop_last=False, - batch_size=train_common_params['data.batch_size'], - num_workers=train_common_params['data.validation_num_workers'], - collate_fn=validation_dataset.collate_fn) - lgr.info(f'Validation Data: Done', {'attrs': 'bold'}) - - # ============================================================================== - # Model - # ============================================================================== - lgr.info('Model:', {'attrs': 'bold'}) - - model = Fuse_model_3d_multichannel( - conv_inputs=(('data.input', 1),), - backbone= ResNet(ch_num=TRAIN_COMMON_PARAMS['backbone_model_dict']['input_channels_num']), - # since backbone resnet contains pooling and fc, the feature output is 1D, - # hence we use Head1DClassifier as classification head - heads=[ - Head1DClassifier(head_name='isLargeTumorSize', - conv_inputs=[('model.backbone_features', train_common_params['num_backbone_features'])], - post_concat_inputs = train_common_params['post_concat_inputs'], - post_concat_model = train_common_params['post_concat_model'], - dropout_rate=0.25, - shared_classifier_head=None, - layers_description=None, - num_classes=2), - - ] - ) - lgr.info('Model: Done', {'attrs': 'bold'}) - - # ==================================================================================== - # Loss - # ==================================================================================== - lgr.info('Losses: CrossEntropy', {'attrs': 'bold'}) - - losses = { - 'cls_loss': LossDefault(pred='model.logits.isLargeTumorSize', - target='data.ground_truth', - callable=F.cross_entropy, weight=1.0), - } - - - # ==================================================================================== - # Metrics - # ==================================================================================== - lgr.info('Metrics:', {'attrs': 'bold'}) - - metrics = { - - 'auc': MetricAUCROC(pred='model.output.isLargeTumorSize', target='data.ground_truth', - class_names=train_common_params['task'].class_names()), - } - - - ()# ===================================================================================== - # Callbacks - # ===================================================================================== - callbacks = [ - TensorboardCallback(model_dir=paths['model_dir']), # save statistics for tensorboard - MetricStatisticsCallback(output_path=paths['model_dir'] + "/metrics.csv"), - # save statistics for tensorboard in a csv file - TimeStatisticsCallback(num_epochs=train_common_params['manager.train_params']['num_epochs'], - load_expected_part=0.1) # time profiler - ] - - # ===================================================================================== - # Manager - Train - # ===================================================================================== - lgr.info('Train:', {'attrs': 'bold'}) - - # create optimizer - optimizer = optim.Adam(model.parameters(), - lr=train_common_params['manager.learning_rate'], - weight_decay=train_common_params['manager.weight_decay']) - # - # create scheduler - scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True) - - # train from scratch - manager = ManagerDefault(output_model_dir=paths['model_dir'], force_reset=paths['force_reset_model_dir']) - # Providing the objects required for the training process. - manager.set_objects(net=model, - optimizer=optimizer, - losses=losses, - metrics=metrics, - best_epoch_source=train_common_params['manager.best_epoch_source'], - lr_scheduler=scheduler, - callbacks=callbacks, - train_params=train_common_params['manager.train_params']) - - ## Continue training - if train_common_params['manager.resume_checkpoint_filename'] is not None: - # Loading the checkpoint including model weights, learning rate, and epoch_index. - manager.load_checkpoint(checkpoint=train_common_params['manager.resume_checkpoint_filename'], mode='train') - - # Start training - manager.train(train_dataloader=train_dataloader, - validation_dataloader=validation_dataloader) - - lgr.info('Train: Done', {'attrs': 'bold'}) - - - -###################################### -# Inference Common Params -###################################### -INFER_COMMON_PARAMS = {} -INFER_COMMON_PARAMS['partition_version'] = '11102021TumorSize' -INFER_COMMON_PARAMS['db_name'] = 'DUKE' -INFER_COMMON_PARAMS['fold_no'] = 0 -INFER_COMMON_PARAMS['infer_filename'] = os.path.join(PATHS['inference_dir'], 'validation_set_infer.pickle.gz') -INFER_COMMON_PARAMS['checkpoint'] = 'best' # Fuse TIP: possible values are 'best', 'last' or epoch_index. - -###################################### -# Inference Template -###################################### -def infer_template(paths: dict, infer_common_params: dict): - #### Logger - # fuse_logger_start(output_path=paths['inference_dir'], console_verbose_level=logging.INFO) - lgr = logging.getLogger('Fuse') - lgr.info('Fuse Inference', {'attrs': ['bold', 'underline']}) - lgr.info(f'infer_filename={infer_common_params["infer_filename"]}', {'color': 'magenta'}) - lgr.info(f'db_name={infer_common_params["db_name"]}', {'color': 'magenta'}) - - #### create dataloader - - ## Create data source: - infer_data_source = ProstateXDataSourcePatient(paths['data_dir'],'validation', - db_ver=infer_common_params['partition_version'], - db_name = infer_common_params['db_name'], - fold_no=infer_common_params['fold_no']) - - - lgr.info(f'db_name={infer_common_params["db_name"]}', {'color': 'magenta'}) - ### load dataset - data_set_filename = os.path.join(paths["model_dir"], "inference_dataset.pth") - dataset = DatasetBase.load(filename=data_set_filename, override_datasource=infer_data_source, override_cache_dest=paths["cache_dir"], num_workers=0) - dataloader = DataLoader(dataset=dataset, - shuffle=False, - drop_last=False, - batch_size=50, - num_workers=5, - collate_fn=dataset.collate_fn) - - #### Manager for inference - manager = ManagerDefault() - # extract just the global classification per sample and save to a file - output_columns = ['model.output.isLargeTumorSize','data.ground_truth'] - manager.infer(data_loader=dataloader, - input_model_dir=paths['model_dir'], - checkpoint=infer_common_params['checkpoint'], - output_columns=output_columns, - output_file_name=infer_common_params['infer_filename']) - -###################################### -# Analyze Common Params -###################################### -EVAL_COMMON_PARAMS = {} -EVAL_COMMON_PARAMS['infer_filename'] = INFER_COMMON_PARAMS['infer_filename'] - -###################################### -# Analyze Template -###################################### -def eval_template(paths: dict, eval_common_params: dict): - fuse_logger_start(output_path=None, console_verbose_level=logging.INFO) - lgr = logging.getLogger('Fuse') - lgr.info('Fuse Eval', {'attrs': ['bold', 'underline']}) - - # metrics - metrics = { - 'roc': MetricROCCurve(pred='model.output.isLargeTumorSize', target='data.ground_truth', - output_filename=os.path.join(paths['inference_dir'], 'roc_curve.png')), - 'auc': MetricAUCROC(pred='model.output.isLargeTumorSize', target='data.ground_truth') - } - - # create evaluator - evaluator = EvaluatorDefault() - - # run - results = evaluator.eval(ids=None, - data=eval_common_params["infer_filename"], - metrics=metrics, - output_dir=paths['eval_dir']) - - return results -###################################### -# Run -###################################### -if __name__ == "__main__": - - # allocate gpus - NUM_GPUS = 1 - if NUM_GPUS == 0: - TRAIN_COMMON_PARAMS['manager.train_params']['device'] = 'cpu' - # uncomment if you want to use specific gpus instead of automatically looking for free ones - force_gpus = [1] # [0] - GPU.choose_and_enable_multiple_gpus(NUM_GPUS, force_gpus=force_gpus) - - RUNNING_MODES = ['train', 'infer', 'eval'] # Options: 'train', 'infer', 'eval' - - if 'train' in RUNNING_MODES: - train_template(paths=PATHS, train_common_params=TRAIN_COMMON_PARAMS) - - if 'infer' in RUNNING_MODES: - infer_template(paths=PATHS, infer_common_params=INFER_COMMON_PARAMS) - - if 'eval' in RUNNING_MODES: - eval_template(paths=PATHS,eval_common_params=EVAL_COMMON_PARAMS) \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/runner_duke.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/runner_duke.py new file mode 100644 index 000000000..c47c4472a --- /dev/null +++ b/examples/fuse_examples/imaging/classification/duke_breast_cancer/runner_duke.py @@ -0,0 +1,488 @@ +""" +(C) Copyright 2021 IBM Corp. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Created on June 30, 2021 + +""" + +import logging +import os +from typing import OrderedDict, Optional + +import torch.nn.functional as F +import torch.optim as optim +from torch.utils.data.dataloader import DataLoader + +import fuse.utils.gpu as GPU +import fuseimg.datasets.duke_label_type +from fuse_fuse_examples_utils import ask_user +from fuse_examples.imaging.classification import duke_breast_cancer +from fuse_examples.imaging.utils.backbone_3d_multichannel import Fuse_model_3d_multichannel, ResNet +from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.samplers import BatchSamplerDefault +from fuse.data.utils.split import dataset_balanced_division_to_folds +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.managers.callbacks.callback_metric_statistics import MetricStatisticsCallback +from fuse.dl.managers.callbacks.callback_tensorboard import TensorboardCallback +from fuse.dl.managers.callbacks.callback_time_statistics import TimeStatisticsCallback +from fuse.dl.managers.manager_default import ManagerDefault +from fuse.dl.models.heads import Head1DClassifier +from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.classification.metrics_classification_common import MetricAccuracy, MetricAUCROC, MetricROCCurve +from fuse.eval.metrics.classification.metrics_thresholding_common import MetricApplyThresholds +from fuse.utils.file_io.file_io import load_pickle +from fuse.utils.rand.seed import Seed +from fuse.utils.utils_debug import FuseDebug +from fuse.utils.utils_logger import fuse_logger_start +from fuseimg.datasets import duke + + +def main(): + mode = 'default' # Options: 'default', 'fast', 'debug', 'verbose', 'user'. See details in FuseDebug + + # allocate gpus + # To use cpu - set NUM_GPUS to 0 + if mode == 'debug': + NUM_GPUS = 1 + else: + NUM_GPUS = 2 + # uncomment if you want to use specific gpus instead of automatically looking for free ones + force_gpus = None # [0] + GPU.choose_and_enable_multiple_gpus(NUM_GPUS, force_gpus=force_gpus) + + if False: + selected_positive = [1, 2, 3, 5, 6, 10, 12, 596, 900, 901] + selected_negative = [4, 6, 7, 8, 11, 13, 14, 120, 902, 903] + + selected_sample_ids = [f'Breast_MRI_{ii:03d}' for ii in selected_positive + selected_negative] + else: + selected_sample_ids = None + + PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS = get_setting(mode, selected_sample_ids=selected_sample_ids) + print(PATHS) + + RUNNING_MODES = ['train', 'infer', 'eval'] # Options: 'train', 'infer', 'eval' + # train + if 'train' in RUNNING_MODES: + print(TRAIN_COMMON_PARAMS) + run_train(paths=PATHS, train_params=TRAIN_COMMON_PARAMS) + + # infer + if 'infer' in RUNNING_MODES: + print(INFER_COMMON_PARAMS) + run_infer(paths=PATHS, infer_common_params=INFER_COMMON_PARAMS, audit_cache='train' not in RUNNING_MODES) + + # eval + if 'eval' in RUNNING_MODES: + print(EVAL_COMMON_PARAMS) + run_eval(paths=PATHS, eval_common_params=EVAL_COMMON_PARAMS) + + print(f"Done running with heldout={INFER_COMMON_PARAMS['data.infer_folds']}") + +def get_setting(mode, label_type=fuseimg.datasets.duke_label_type.DukeLabelType.STAGING_TUMOR_SIZE, n_folds=5, heldout_fold=4, + selected_sample_ids=None, num_epoch=None): + ########################################################################################################### + # Fuse + ########################################################################################################### + ########################################## + # Debug modes + ########################################## + + debug = FuseDebug(mode) + ########################################## + # Output Paths + ########################################## + assert "DUKE_DATA_PATH" in os.environ, "Expecting environment variable DUKE_DATA_PATH to be set. Follow the instruction in example README file to download and set the path to the data" + ROOT = duke_breast_cancer.get_duke_user_dir() + + data_dir = os.environ["DUKE_DATA_PATH"] + + if mode == 'debug': + data_split_file = os.path.join(ROOT, 'DUKE_folds_debug.pkl') + if selected_sample_ids is None: + selected_sample_ids = duke.get_samples_for_debug(data_dir=data_dir, n_pos=10, n_neg=10, label_type=label_type, + sample_ids=duke.get_selected_sample_ids()) + print(selected_sample_ids) + cache_dir = os.path.join(ROOT, 'cache_dir_debug') + model_dir = os.path.join(ROOT, 'model_dir_debug') + num_workers = 0 + batch_size = 2 + if num_epoch is None: + num_epoch = 5 + else: + data_split_file = os.path.join(ROOT, 'DUKE_folds_v5.pkl') + cache_dir = os.path.join(ROOT, 'cache_dir_v5') + model_dir = os.path.join(ROOT, 'model_dir_v5') + + num_workers = 16 # put 0 to debug + batch_size = 50 + if num_epoch is None: + num_epoch = 20 # 150 + + PATHS = {'model_dir': model_dir, + 'force_reset_model_dir': True, # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. + 'cache_dir': cache_dir, + 'data_split_filename': os.path.join(ROOT, data_split_file), + 'data_dir': data_dir, + 'inference_dir': os.path.join(model_dir, 'infer_dir'), + 'eval_dir': os.path.join(model_dir, 'eval_dir'), + } + + ########################################## + # Train Common Params + ########################################## + TRAIN_COMMON_PARAMS = {} + # ============ + # Model + # ============ + + # ============ + # Data + # ============ + + train_folds = [i % n_folds for i in range(heldout_fold + 1, heldout_fold + n_folds - 1)] + validation_fold = (heldout_fold - 1) % n_folds + TRAIN_COMMON_PARAMS['data.selected_sample_ids'] = selected_sample_ids + TRAIN_COMMON_PARAMS['data.batch_size'] = batch_size + TRAIN_COMMON_PARAMS['data.train_num_workers'] = num_workers + TRAIN_COMMON_PARAMS['data.validation_num_workers'] = num_workers + TRAIN_COMMON_PARAMS['data.num_folds'] = n_folds + TRAIN_COMMON_PARAMS['data.train_folds'] = train_folds + TRAIN_COMMON_PARAMS['data.validation_folds'] = [validation_fold] + + # =============== + # Manager - Train + # =============== + TRAIN_COMMON_PARAMS['manager.train_params'] = { + 'num_epochs': num_epoch, + 'virtual_batch_size': 1, # number of batches in one virtual batch + 'start_saving_epochs': 10, # first epoch to start saving checkpoints from + 'gap_between_saving_epochs': 5, # number of epochs between saved checkpoint + } + TRAIN_COMMON_PARAMS['manager.best_epoch_source'] = { + 'source': 'metrics.auc', # can be any key from 'epoch_results' + 'optimization': 'max', # can be either min/max + 'on_equal_values': 'better', + # can be either better/worse - whether to consider best epoch when values are equal + } + TRAIN_COMMON_PARAMS['manager.learning_rate'] = 1e-5 + TRAIN_COMMON_PARAMS['manager.weight_decay'] = 0.001 + TRAIN_COMMON_PARAMS['manager.dropout'] = 0.5 + TRAIN_COMMON_PARAMS['manager.momentum'] = 0.9 + TRAIN_COMMON_PARAMS['manager.resume_checkpoint_filename'] = None # if not None, will try to load the checkpoint + TRAIN_COMMON_PARAMS['imaging_dropout'] = 0.25 + # TRAIN_COMMON_PARAMS['fused_dropout'] = 0.0 + # TRAIN_COMMON_PARAMS['clinical_dropout'] = 0.0 + + TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] = 512 + + # in order to add relevant tabular feature uncomment: + # num_backbone_features_clinical, post_concat_inputs,post_concat_model + TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] = None # 256 + TRAIN_COMMON_PARAMS['post_concat_inputs'] = None # [('data.clinical_features',9),] + TRAIN_COMMON_PARAMS['post_concat_model'] = None # (256,256) + + if TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] is None: + TRAIN_COMMON_PARAMS['num_backbone_features'] = TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] + else: + TRAIN_COMMON_PARAMS['num_backbone_features'] = \ + TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] + TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] + + # classification_task: + # supported tasks are: 'Staging Tumor Size','Histology Type','is High Tumor Grade Total','PCR' + TRAIN_COMMON_PARAMS['classification_task'] = label_type + TRAIN_COMMON_PARAMS['class_num'] = label_type.get_num_classes() + + # backbone parameters + TRAIN_COMMON_PARAMS['backbone_model_dict'] = \ + {'input_channels_num': 1, + } + + ###################################### + # Inference Common Params + ###################################### + INFER_COMMON_PARAMS = {} + INFER_COMMON_PARAMS['infer_filename'] = 'validation_set_infer.gz' + INFER_COMMON_PARAMS['checkpoint'] = 'best' # Fuse TIP: possible values are 'best', 'last' or epoch_index. + INFER_COMMON_PARAMS['data.infer_folds'] = [heldout_fold] # infer validation set + INFER_COMMON_PARAMS['data.batch_size'] = 4 + INFER_COMMON_PARAMS['data.num_workers'] = num_workers + INFER_COMMON_PARAMS['classification_task'] = TRAIN_COMMON_PARAMS['classification_task'] + + ###################################### + # Analyze Common Params + ###################################### + EVAL_COMMON_PARAMS = {} + EVAL_COMMON_PARAMS['infer_filename'] = INFER_COMMON_PARAMS['infer_filename'] + + return PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS + + +################################# +# Train Template +################################# +def run_train(paths: dict, train_params: dict, reset_cache=None, audit_cache=None): + Seed.set_seed(222, False) + # ============================================================================== + # Logger + # ============================================================================== + fuse_logger_start(output_path=paths['model_dir'], console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Train', {'attrs': ['bold', 'underline']}) + + lgr.info(f'model_dir={paths["model_dir"]}', {'color': 'magenta'}) + lgr.info(f'cache_dir={paths["cache_dir"]}', {'color': 'magenta'}) + lgr.info(f'train folds={train_params["data.train_folds"]}', {'color': 'magenta'}) + lgr.info(f'validation folds={train_params["data.validation_folds"]}', {'color': 'magenta'}) + + + # ============================================================================== + # Data + # ============================================================================== + # Train Data + lgr.info(f'Train Data:', {'attrs': 'bold'}) + + if reset_cache is None: + reset_cache = ask_user('Do you want to reset cache?') + cache_kwargs = {'use_pipeline_hash': False} + if not reset_cache: + if audit_cache is None: + audit_cache = ask_user('Do you want to audit cache?') + if not audit_cache: + cache_kwargs2 = dict(audit_first_sample=False, audit_rate=None) + cache_kwargs = {**cache_kwargs, **cache_kwargs2} + + # split to folds randomly + params = dict(label_type=train_params['classification_task'], data_dir=paths["data_dir"], cache_dir=paths["cache_dir"], + reset_cache=reset_cache, sample_ids=train_params['data.selected_sample_ids'], + num_workers=train_params['data.train_num_workers'], + cache_kwargs=cache_kwargs, train=False, verbose=False) + + dataset_all = duke.Duke.dataset(**params) + folds = dataset_balanced_division_to_folds(dataset=dataset_all, + output_split_filename=paths["data_split_filename"], + keys_to_balance=["data.ground_truth"], + workers=0, # todo: stuck in Export to dataframe + nfolds=train_params["data.num_folds"], + verbose=True) + + train_sample_ids = [] + for fold in train_params["data.train_folds"]: + train_sample_ids += folds[fold] + validation_sample_ids = [] + for fold in train_params["data.validation_folds"]: + validation_sample_ids += folds[fold] + + params['sample_ids'] = train_sample_ids + params['reset_cache'] = False + params['train'] = True + params['cache_kwargs'] = dict(use_pipeline_hash=False, audit_first_sample=False, audit_rate=None) + train_dataset = duke.Duke.dataset(**params) + # for _ in train_dataset: + # pass + params['sample_ids'] = validation_sample_ids + params['train'] = False + validation_dataset = duke.Duke.dataset(**params) + + lgr.info(f'- Create sampler:') + sampler = BatchSamplerDefault(dataset=train_dataset, + balanced_class_name='data.ground_truth', + num_balanced_classes=train_params['class_num'], + batch_size=train_params['data.batch_size'], + workers=0 #train_params['data.train_num_workers'] #todo: stuck + ) + lgr.info(f'- Create sampler: Done') + + # Create dataloader + train_dataloader = DataLoader(dataset=train_dataset, + batch_sampler=sampler, + collate_fn=CollateDefault(), + num_workers=train_params['data.train_num_workers']) + lgr.info(f'Train Data: Done', {'attrs': 'bold'}) + + # dataloader + validation_dataloader = DataLoader(dataset=validation_dataset, + batch_size=train_params['data.batch_size'], + collate_fn=CollateDefault(), + num_workers=train_params['data.validation_num_workers']) + lgr.info(f'Validation Data: Done', {'attrs': 'bold'}) + + # ============================================================================== + # Model + # ============================================================================== + lgr.info('Model:', {'attrs': 'bold'}) + + conv_inputs = (('data.input.patch_volume', 1),) + model = Fuse_model_3d_multichannel( + conv_inputs=conv_inputs, # previously 'data.input'. could be either 'data.input.patch_volume' or 'data.input.patch_volume_orig' + backbone=ResNet(conv_inputs=conv_inputs, ch_num=train_params['backbone_model_dict']['input_channels_num']), + # since backbone resnet contains pooling and fc, the feature output is 1D, + # hence we use Head1dClassifier as classification head + heads=[ + Head1DClassifier(head_name='classification', + conv_inputs=[('model.backbone_features', train_params['num_backbone_features'])], + post_concat_inputs=train_params['post_concat_inputs'], + post_concat_model=train_params['post_concat_model'], + dropout_rate=train_params['imaging_dropout'], + # append_dropout_rate=train_params['clinical_dropout'], + # fused_dropout_rate=train_params['fused_dropout'], + shared_classifier_head=None, + layers_description=None, + num_classes=2, + # append_features=[("data.input.clinical", 8)], + # append_layers_description=(256,128), + ), + ] + ) + + lgr.info('Model: Done', {'attrs': 'bold'}) + + # ==================================================================================== + # Loss + # ==================================================================================== + losses = { + 'cls_loss': LossDefault(pred='model.logits.classification', + target='data.ground_truth', callable=F.cross_entropy, weight=1.0), + } + + # ==================================================================================== + # Metrics + # ==================================================================================== + lgr.info('Metrics:', {'attrs': 'bold'}) + metrics = OrderedDict([ + ('auc', MetricAUCROC(pred='model.output.classification', target='data.ground_truth')) + ]) + + # ===================================================================================== + # Callbacks + # ===================================================================================== + callbacks = [ + # default callbacks + TensorboardCallback(model_dir=paths['model_dir']), # save statistics for tensorboard + MetricStatisticsCallback(output_path=paths['model_dir'] + "/metrics.csv"), # save statistics a csv file + TimeStatisticsCallback(num_epochs=train_params['manager.train_params']['num_epochs'], load_expected_part=0.1) # time profiler + ] + + # ===================================================================================== + # Manager - Train + # ===================================================================================== + lgr.info('Train:', {'attrs': 'bold'}) + + # create optimizer + optimizer = optim.Adam(model.parameters(), lr=train_params['manager.learning_rate'], + weight_decay=train_params['manager.weight_decay']) + + # create learning scheduler + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True) + + # train from scratch + manager = ManagerDefault(output_model_dir=paths['model_dir'], force_reset=paths['force_reset_model_dir']) + # Providing the objects required for the training process. + manager.set_objects(net=model, + optimizer=optimizer, + losses=losses, + metrics=metrics, + best_epoch_source=train_params['manager.best_epoch_source'], + lr_scheduler=scheduler, + callbacks=callbacks, + train_params=train_params['manager.train_params']) + + ## Continue training + if train_params['manager.resume_checkpoint_filename'] is not None: + # Loading the checkpoint including model weights, learning rate, and epoch_index. + manager.load_checkpoint(checkpoint=train_params['manager.resume_checkpoint_filename'], mode='train') + + # Start training + manager.train(train_dataloader=train_dataloader, validation_dataloader=validation_dataloader) + + lgr.info('Train: Done', {'attrs': 'bold'}) + + +###################################### +# Inference Template +###################################### +def run_infer(paths: dict, infer_common_params: dict, audit_cache: Optional[bool] = True): + #### Logger + fuse_logger_start(output_path=paths['inference_dir'], console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Inference', {'attrs': ['bold', 'underline']}) + lgr.info(f'infer_filename={os.path.join(paths["inference_dir"], infer_common_params["infer_filename"])}', {'color': 'magenta'}) + lgr.info(f'infer folds={infer_common_params["data.infer_folds"]}', {'color': 'magenta'}) + + ## Data + folds = load_pickle(paths["data_split_filename"]) # assume exists and created in train func + + infer_sample_ids = [] + for fold in infer_common_params["data.infer_folds"]: + infer_sample_ids += folds[fold] + + params = dict(label_type=infer_common_params['classification_task'], data_dir=paths["data_dir"], + cache_dir=paths["cache_dir"], train=False, + sample_ids=infer_sample_ids, + verbose=False) + if not audit_cache: + params['cache_kwargs'] =dict(use_pipeline_hash=False, audit_first_sample=False, audit_rate=None) + else: + params['cache_kwargs'] = dict(use_pipeline_hash=False) + validation_dataset = duke.Duke.dataset(**params) + + # dataloader + validation_dataloader = DataLoader(dataset=validation_dataset, batch_size=infer_common_params['data.batch_size'], collate_fn=CollateDefault(), + num_workers=infer_common_params['data.num_workers']) + + ## Manager for inference + manager = ManagerDefault() + output_columns = ['model.output.classification', 'data.ground_truth'] + manager.infer(data_loader=validation_dataloader, + input_model_dir=paths['model_dir'], + checkpoint=infer_common_params['checkpoint'], + output_columns=output_columns, + output_file_name=os.path.join(paths["inference_dir"], infer_common_params["infer_filename"])) + + +###################################### +# Analyze Template +###################################### +def run_eval(paths: dict, eval_common_params: dict): + fuse_logger_start(output_path=None, console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Analyze', {'attrs': ['bold', 'underline']}) + + # metrics + metrics = OrderedDict([ + ('operation_point', MetricApplyThresholds(pred='model.output.classification')), # will apply argmax + ('accuracy', MetricAccuracy(pred='results:metrics.operation_point.cls_pred', target='data.ground_truth')), + ('roc', MetricROCCurve(pred='model.output.classification', target='data.ground_truth', + output_filename=os.path.join(paths['inference_dir'], 'roc_curve.png'))), + ('auc', MetricAUCROC(pred='model.output.classification', target='data.ground_truth')), + ]) + + # create evaluator + evaluator = EvaluatorDefault() + + # run + results = evaluator.eval(ids=None, + data=os.path.join(paths["inference_dir"], eval_common_params["infer_filename"]), + metrics=metrics, + output_dir=paths['eval_dir']) + + return results + + +###################################### +# Run +###################################### +if __name__ == "__main__": + main() diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/runner_duke_radiomics.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/runner_duke_radiomics.py new file mode 100644 index 000000000..f72f3259b --- /dev/null +++ b/examples/fuse_examples/imaging/classification/duke_breast_cancer/runner_duke_radiomics.py @@ -0,0 +1,266 @@ +import os +import numpy as np + +import fuseimg.datasets.duke_label_type + +import getpass +from fuseimg.datasets import duke +from fuse.utils.utils_debug import FuseDebug +import logging +from fuse.utils.utils_logger import fuse_logger_start +from fuse.data.utils.split import dataset_balanced_division_to_folds +from typing import OrderedDict +from fuse.eval.metrics.classification.metrics_classification_common import MetricAccuracy, MetricAUCROC, MetricROCCurve + +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import LogisticRegression + + +def main(): + mode = 'debug' #'default' # 'default' # Options: 'default', 'fast', 'debug', 'verbose', 'user'. See details in FuseDebug + + PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS = get_setting(mode) + print(PATHS) + + RUNNING_MODES = ['train', 'infer', 'eval'] # Options: 'train', 'infer', 'eval' + # train + if 'train' in RUNNING_MODES: + print(TRAIN_COMMON_PARAMS) + run_train(paths=PATHS, train_params=TRAIN_COMMON_PARAMS) + + # # infer + # if 'infer' in RUNNING_MODES: + # print(INFER_COMMON_PARAMS) + # run_infer(paths=PATHS, infer_common_params=INFER_COMMON_PARAMS) + # + # # eval + # if 'eval' in RUNNING_MODES: + # print(EVAL_COMMON_PARAMS) + # run_eval(paths=PATHS, eval_common_params=EVAL_COMMON_PARAMS) + + +def get_setting(mode, label_type=fuseimg.datasets.duke_label_type.DukeLabelType.STAGING_TUMOR_SIZE, n_folds=5, heldout_fold=4): + ########################################################################################################### + # Fuse + ########################################################################################################### + ########################################## + # Debug modes + ########################################## + + debug = FuseDebug(mode) + + + + + ########################################## + # Output Paths + ########################################## + assert "DUKE_DATA_PATH" in os.environ, "Expecting environment variable DUKE_DATA_PATH to be set. Follow the instruction in example README file to download and set the path to the data" + ROOT = f'/projects/msieve_dev3/usr/{getpass.getuser()}/fuse_examples/duke_radiomics' + model_dir = os.path.join(ROOT, 'model_dir') + + if mode == 'debug': + num_workers = 16 #0 + selected_sample_ids = duke.get_samples_for_debug(n_pos=10, n_neg=10, label_type=label_type) + cache_dir = os.path.join(ROOT, 'cache_dir_debug') + data_split_file = os.path.join(ROOT, 'DUKE_radiomics_folds_debug.pkl') + + else: + num_workers = 16 + selected_sample_ids = None + cache_dir = os.path.join(ROOT, 'cache_dir') + data_split_file = os.path.join(ROOT, 'DUKE_radiomics_folds.pkl') + + + cache_dir = os.path.join(ROOT, cache_dir) + + PATHS = {'model_dir': model_dir, + 'force_reset_model_dir': True, # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. + 'cache_dir': cache_dir, + 'data_split_filename': os.path.join(ROOT, data_split_file), + 'data_dir': os.environ["DUKE_DATA_PATH"], + 'inference_dir': os.path.join(model_dir, 'infer_dir'), + 'eval_dir': os.path.join(model_dir, 'eval_dir'), + } + + ########################################## + # Train Common Params + ########################################## + TRAIN_COMMON_PARAMS = {} + # ============ + # Model + # ============ + + # ============ + # Data + # ============ + + train_folds = [i % n_folds for i in range(heldout_fold + 1, heldout_fold + n_folds - 1)] + validation_fold = (heldout_fold - 1) % n_folds + TRAIN_COMMON_PARAMS['data.selected_sample_ids'] = selected_sample_ids + TRAIN_COMMON_PARAMS['data.num_folds'] = n_folds + TRAIN_COMMON_PARAMS['data.train_folds'] = train_folds + TRAIN_COMMON_PARAMS['data.validation_folds'] = [validation_fold] + TRAIN_COMMON_PARAMS['data.train_num_workers'] = num_workers + + def get_selected_series_index_radiomics(sample_id, seq_id): + patient_id = sample_id[0] + if patient_id in ['Breast_MRI_120', 'Breast_MRI_596']: + map = {'DCE_mix': [1, 2], 'MASK': [0]} + else: + map = {'DCE_mix': [0, 1], 'MASK': [0]} + return map[seq_id] + TRAIN_COMMON_PARAMS['data.get_selectedseries_index_func'] = get_selected_series_index_radiomics + + TRAIN_COMMON_PARAMS['radiomics_extractor_setting'] = get_radiomics_extractor_setting2() + + # classification_task: + # supported tasks are: 'Staging Tumor Size','Histology Type','is High Tumor Grade Total','PCR' + TRAIN_COMMON_PARAMS['classification_task'] = label_type + TRAIN_COMMON_PARAMS['models'] = [('lr', LogisticRegression()), ('rf', RandomForestClassifier())] + + ###################################### + # Inference Common Params + ###################################### + INFER_COMMON_PARAMS = {} + INFER_COMMON_PARAMS['infer_filename'] = 'radiomics_validation_set_infer.gz' + INFER_COMMON_PARAMS['data.infer_folds'] = [heldout_fold] # infer validation set + INFER_COMMON_PARAMS['classification_task'] = TRAIN_COMMON_PARAMS['classification_task'] + + ###################################### + # Analyze Common Params + ###################################### + EVAL_COMMON_PARAMS = {} + EVAL_COMMON_PARAMS['infer_filename'] = INFER_COMMON_PARAMS['infer_filename'] + + return PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS + + +def run_train(paths: dict, train_params: dict, reset_cache=False): + # ============================================================================== + # Logger + # ============================================================================== + fuse_logger_start(output_path=paths['model_dir'], console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Train', {'attrs': ['bold', 'underline']}) + + lgr.info(f'model_dir={paths["model_dir"]}', {'color': 'magenta'}) + lgr.info(f'cache_dir={paths["cache_dir"]}', {'color': 'magenta'}) + + # ============================================================================== + # Data + # ============================================================================== + # Train Data + lgr.info(f'Train Data:', {'attrs': 'bold'}) + + + # split to folds randomly - temp + params = dict(label_type=train_params['classification_task'], data_dir=paths["data_dir"], + cache_dir=paths["cache_dir"], + reset_cache=False, + sample_ids=train_params['data.selected_sample_ids'], num_workers=train_params['data.train_num_workers'], + radiomics_extractor_setting=train_params['radiomics_extractor_setting'], + select_series_func=train_params['data.get_selectedseries_index_func'], + cache_kwargs= dict(audit_first_sample=False, audit_rate=None) # None + ) + dataset_all = duke.DukeRadiomics.dataset(**params) + folds = dataset_balanced_division_to_folds(dataset=dataset_all, + output_split_filename=paths["data_split_filename"], + keys_to_balance=["data.ground_truth"], + nfolds=train_params["data.num_folds"]) + + print("---------------") + train_sample_ids = [] + for fold in train_params["data.train_folds"]: + train_sample_ids += folds[fold] + validation_sample_ids = [] + for fold in train_params["data.validation_folds"]: + validation_sample_ids += folds[fold] + + params['sample_ids'] = train_sample_ids + train_dataset = duke.DukeRadiomics.dataset(**params) + + params['sample_ids']=validation_sample_ids + # validation_dataset = duke.DukeRadiomics.dataset(**params) + + # ============================================================================== + # Model + # ============================================================================== + + # model1 = RandomForestClassifier() + # model2 = LogisticRegression() + X_train, xnames, y_train = get_X_y(train_dataset) + print(X_train.shape, len(xnames), y_train.shape) + print("#nans=",np.isnan(X_train).sum().sum()) + # + # X_val, y_val = get_X_y(validation_dataset) + + lgr.info('Train: Done', {'attrs': 'bold'}) + + +def get_radiomics_extractor_setting(norm_method='default'): + setting = {} + + setting['seq_vec'] = ['DCE'] + setting['seq_inx_list'] = [0] + setting['maskType'] = 'full' #alternatives: 'edge' + setting['norm_method'] = norm_method # alternative: 'tumor_area', 'breast_area' + + if norm_method == 'default': + setting['normalize'] = True + setting['normalizeScale'] = 100 + else: + setting['normalize'] = False + + setting['binWidth'] = 5 + setting['preCrop'] = True + setting['applyLog'] = False + setting['applyWavelet'] = False + +def get_radiomics_extractor_setting2(norm_method='default'): + setting ={} + setting['seq_list'] = ['DCE0', 'DCE1'] + setting['seq_inx_list'] = [0, 1] + setting['norm_method'] = norm_method + setting['maskType'] = 'full' + + if norm_method == 'default': + setting['normalize'] = True + setting['normalizeScale'] = 100 + else: + setting['normalize'] = False + + setting['binWidth'] = 5 + setting['preCrop'] = True + setting['applyLog'] = False + setting['applyWavelet'] = False + ### + + return setting + + +def get_X_y(a_dataset): + feature_keys = fnames = None + y_list = [] + X_list = [] + for i, d in enumerate(a_dataset): + if i==0: + feature_keys = sorted([s for s in d.flatten() if s.startswith('data.radiomics')]) + fnames = [s.replace('data.radiomics.', '') for s in feature_keys] + is_ndarray = [isinstance(d[k], np.ndarray) and len(d[k].shape)>0 for k in feature_keys] + + y = d['data.ground_truth'] + + X = np.asarray([d[k][0] if flag else float(d[k]) for k, flag in zip(feature_keys, is_ndarray)]) + y_list.append(y) + X_list.append(X) + + y = np.asarray(y_list) + X = np.asarray(X_list) + print(y.shape, X.shape) + return X, fnames, y + + + +if __name__ == '__main__': + main() diff --git a/examples/fuse_examples/imaging/classification/duke_breast_cancer/tasks.py b/examples/fuse_examples/imaging/classification/duke_breast_cancer/tasks.py deleted file mode 100644 index fc3437deb..000000000 --- a/examples/fuse_examples/imaging/classification/duke_breast_cancer/tasks.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -import logging -from typing import List - - -class Task(): - tasks = {} - def __init__(self, task_name: str, version: int): - self._task_name, self._task_version, self._task_mapping, self._task_class_names = \ - self.get_task(task_name, version) - - def name(self): - return self._task_name + "_" + str(self._task_version) - - def class_names(self): - return self._task_class_names - - def num_classes(self): - return len(self._task_class_names) - - def mapping(self): - return self._task_mapping - - @classmethod - def register(cls, name: str, version: int, mapping: List, class_names: List[str]): - key = (name, version) - assert key not in cls.tasks - cls.tasks[key] = (name, version, mapping, class_names) - - @classmethod - def get_task(cls, task_name: str, version: int): - key = (task_name, version) - if key not in cls.tasks: - msg = f'Task not found - list of tasks: {list(cls.tasks.keys())}' - logging.getLogger('Fuse').error(msg) - raise Exception(msg) - - return cls.tasks[key] - - - -#DO NOT CHANGE TASKS!!!! - -pcr_SCORE_VER_0 = [['HIGH'], ['LOW']], -tumor_size_VER_0 = [['HIGH'], ['LOW']], -tumor_grade_VER_0 = [['HIGH'], ['LOW']], -histotype_VER_0 = [['HIGH'], ['LOW']], -Task.register('ispCR', 0, pcr_SCORE_VER_0, ['HIGH','LOW']) -Task.register('Histology Type', 0, histotype_VER_0, ['HIGH','LOW']) -Task.register('is High Tumor Grade Total', 0, tumor_grade_VER_0, ['HIGH','LOW']) -Task.register('Staging Tumor Size', 0, tumor_size_VER_0, ['HIGH','LOW']) - -if __name__ == '__main__': - mp_task = Task('gleason_score', 0) - print(mp_task.name()) - print(mp_task.class_names()) - print(len(mp_task.class_names())) - print(mp_task.mapping()) \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/knight/make_predictions_file.py b/examples/fuse_examples/imaging/classification/knight/make_predictions_file.py index 8e3fb666a..20f03218f 100644 --- a/examples/fuse_examples/imaging/classification/knight/make_predictions_file.py +++ b/examples/fuse_examples/imaging/classification/knight/make_predictions_file.py @@ -31,7 +31,7 @@ from fuse.utils.file_io.file_io import save_dataframe from fuse.dl.managers.manager_default import ManagerDefault -from examples.fuse_examples.imaging.classification.knight.eval.eval import TASK1_CLASS_NAMES, TASK2_CLASS_NAMES +from fuse_examples.imaging.classification.knight.eval.eval import TASK1_CLASS_NAMES, TASK2_CLASS_NAMES from baseline.dataset import knight_dataset def make_predictions_file(model_dir: str, diff --git a/examples/fuse_examples/imaging/classification/prostate_x/README.md b/examples/fuse_examples/imaging/classification/prostate_x/README.md index 95a57e6a1..8bd8c8874 100644 --- a/examples/fuse_examples/imaging/classification/prostate_x/README.md +++ b/examples/fuse_examples/imaging/classification/prostate_x/README.md @@ -7,7 +7,8 @@ The project presents lesions classification of Gleason score in prostate. It dem **Dataset** -We used the public SPIE-AAPM-NCI PROSTATEx Challenge dataset: https://wiki.cancerimagingarchive.net/display/Public/SPIE-AAPM-NCI+PROSTATEx+Challenges#23691656d4622c5ad5884bdb876d6d441994da38 It contains 204 patients and 330 lesions for training. The train data was split into 8 folds insuring no patient is overlapped between the different folds. The data was split in a way that saves the frequency of the two classes in each of the folds. Data should be located under "'./PatientData/ProstateX/manifest-A3Y4AE4o5818678569166032044/'" +We used the public SPIE-AAPM-NCI PROSTATEx Challenge dataset: +https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=23691656 **Pre-processing** diff --git a/examples/fuse_examples/imaging/classification/prostate_x/data_utils.py b/examples/fuse_examples/imaging/classification/prostate_x/data_utils.py deleted file mode 100644 index 95434e30c..000000000 --- a/examples/fuse_examples/imaging/classification/prostate_x/data_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -import pickle -import pandas as pd -import os - -class ProstateXUtilsData: - @staticmethod - def get_dataset(path_to_db: str,set_type: str, db_ver: int,db_name: str,fold_no: int): - db_name = os.path.join(path_to_db,f'dataset_{db_name}_folds_ver{db_ver}_seed1.pickle') - with open(db_name, 'rb') as infile: - db = pickle.load(infile) - - - if set_type == 'train': - other_folds = list(set(range(0,len(db)))-set([fold_no])) - for i,f in enumerate(other_folds): - if i==0: - data = db['data_fold' + str(f)] - else: - data = pd.concat([data,db['data_fold' + str(f)]],join='inner') - - elif set_type == 'validation': - data = db['data_fold'+str(fold_no)] - elif set_type == 'test': - data = db['test_data'] - else: - # returns the full set - for i,f in enumerate(range(0,len(db))): - if i==0: - data = db['data_fold' + str(f)] - else: - data = pd.concat([data,db['data_fold' + str(f)]],join='inner') - - # raise Exception(f'Unexpected set type {set_type}') - return data - - def get_lesions_prostate_x(data: pd.DataFrame): - outlier_list = [] - lesion_data = data[~data['Patient ID'].isin(outlier_list)] - return lesion_data - - - - -if __name__ == "__main__": - path_to_db = '/gpfs/haifa/projects/m/msieve_dev3/usr/Tal/my_research/virtual_biopsy/prostate/experiments/V1/' - # data = CAPVUtilsData.get_dataset(path_to_db=path_to_db,set_type='train', db_ver=18042021,db_name='tcia',fold_no=0) - # data_lesion = CAPVUtilsData.get_lesions(data) - - data = ProstateXUtilsData.get_dataset(path_to_db=path_to_db, set_type='train', db_ver=29042021, db_name='prostate_x',fold_no=0) - data_lesion = ProstateXUtilsData.get_lesions_prostate_x(data) \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/dataset.py b/examples/fuse_examples/imaging/classification/prostate_x/dataset.py deleted file mode 100644 index 121860458..000000000 --- a/examples/fuse_examples/imaging/classification/prostate_x/dataset.py +++ /dev/null @@ -1,201 +0,0 @@ -from functools import partial -from multiprocessing import Manager - -from fuse.data.augmentor.augmentor_default import AugmentorDefault -from fuse.data.augmentor.augmentor_toolbox import unsqueeze_2d_to_3d, aug_op_color, aug_op_affine, squeeze_3d_to_2d, \ - rotation_in_3d -from fuse.data.dataset.dataset_generator import DatasetGenerator - -import fuse.utils.gpu as GPU -from fuse.utils.rand.param_sampler import Uniform, RandInt, RandBool, Choice - -from fuse_examples.imaging.classification.prostate_x.patient_data_source import ProstateXDataSourcePatient -from fuse_examples.imaging.classification.prostate_x.processor import ProstateXPatchProcessor -from fuse_examples.imaging.classification.prostate_x.post_processor import post_processing -from fuse.data.processor.processor_dicom_mri import DicomMRIProcessor - - -def process_mri_series(): - seq_to_use_dict = \ - { - 't2_tse_tra': 'T2', - 't2_tse_tra_Grappa3': 'T2', - 't2_tse_tra_320_p2': 'T2', - - 'ep2d-advdiff-3Scan-high bvalue 100': 'b', - 'ep2d-advdiff-3Scan-high bvalue 500': 'b', - 'ep2d-advdiff-3Scan-high bvalue 1400': 'b', - 'ep2d_diff_tra2x2_Noise0_FS_DYNDISTCALC_BVAL': 'b', - - 'ep2d_diff_tra_DYNDIST': 'b_mix', - 'ep2d_diff_tra_DYNDIST_MIX': 'b_mix', - 'diffusie-3Scan-4bval_fs': 'b_mix', - 'ep2d_DIFF_tra_b50_500_800_1400_alle_spoelen': 'b_mix', - 'diff tra b 50 500 800 WIP511b alle spoelen': 'b_mix', - - 'ep2d_diff_tra_DYNDIST_MIX_ADC': 'ADC', - 'diffusie-3Scan-4bval_fs_ADC': 'ADC', - 'ep2d-advdiff-MDDW-12dir_spair_511b_ADC': 'ADC', - 'ep2d-advdiff-3Scan-4bval_spair_511b_ADC': 'ADC', - 'ep2d_DIFF_tra_b50_500_800_1400_alle_spoelen_ADC': 'ADC', - 'diff tra b 50 500 800 WIP511b alle spoelen_ADC': 'ADC', - 'ADC_S3_1': 'ADC', - 'ep2d_diff_tra_DYNDIST_ADC': 'ADC', - - } - - # patients with special fix - exp_patients = ['ProstateX-0191', 'ProstateX-0148', 'ProstateX-0180'] - seq_to_use = ['T2', 'b', 'b_mix', 'ADC', 'ktrans'] - subseq_to_use = ['T2', 'b400', 'b800', 'ADC', 'ktrans'] - - SER_INX_TO_USE = {} - SER_INX_TO_USE['all'] = {'T2': -1, 'b': [0, 2], 'ADC': 0, 'ktrans': 0} - SER_INX_TO_USE['ProstateX-0148'] = {'T2': 1, 'b': [1, 2], 'ADC': 0, 'ktrans': 0} - SER_INX_TO_USE['ProstateX-0191'] = {'T2': -1, 'b': [0, 0], 'ADC': 0, 'ktrans': 0} - SER_INX_TO_USE['ProstateX-0180'] = {'T2': -1, 'b': [1, 2], 'ADC': 0, 'ktrans': 0} - - # sequences with special fix - B_SER_FIX = ['diffusie-3Scan-4bval_fs', - 'ep2d_DIFF_tra_b50_500_800_1400_alle_spoelen', - 'diff tra b 50 500 800 WIP511b alle spoelen'] - - return seq_to_use_dict, SER_INX_TO_USE, exp_patients,seq_to_use,subseq_to_use - -def prostate_x_dataset(paths,train_common_params,lgr): - #### Train Data - - lgr.info(f'Train Data:', {'attrs': 'bold'}) - - ## Create data source: - DATABASE_REVISION = train_common_params['db_version'] - lgr.info(f'database_revision={DATABASE_REVISION}', {'color': 'magenta'}) - - # create data source - train_data_source = ProstateXDataSourcePatient(paths['data_dir'], 'train', - db_ver=train_common_params['db_version'], - db_name=train_common_params['db_name'], - fold_no=train_common_params['fold_no']) - - ## Create data processors: - image_processing_args = { - 'patch_xy': 74, - 'patch_z': 13, - } - - ## Create data processor - - seq_to_use_dict, SER_INX_TO_USE, \ - exp_patients, seq_to_use, subseq_to_use = process_mri_series() - - generate_processor = ProstateXPatchProcessor( - vol_processor=DicomMRIProcessor(reference_inx=0, - seq_dict=seq_to_use_dict, - seq_to_use=seq_to_use, - subseq_to_use=subseq_to_use, - ser_inx_to_use=SER_INX_TO_USE, - exp_patients=exp_patients), - - path_to_db=paths['data_dir'], - data_path=paths['prostate_data_path'], - ktrans_data_path=paths['ktrans_path'], - db_name=train_common_params['db_name'], - db_version=train_common_params['db_version'], - fold_no=train_common_params['fold_no'], - lsn_shape=(image_processing_args['patch_z'], - image_processing_args['patch_xy'], - image_processing_args['patch_xy']), - ) - - train_post_processor = partial(post_processing) - - # data augmentation (optional) - num_channels = train_common_params['backbone_model_dict']['input_channels_num'] - slice_num = image_processing_args['patch_z'] - - image_channels = [list(range(0, slice_num))] - aug_pipeline = [ - [ - ('data.input',), - rotation_in_3d, - {'z_rot': Uniform(-5.0, 5.0), 'y_rot': Uniform(-5.0, 5.0), 'x_rot': Uniform(-5.0, 5.0)}, - {'apply': RandBool(0.5)} - ], - [ - ('data.input',), - squeeze_3d_to_2d, - {'axis_squeeze': 'z'}, - {} - ], - [ - ('data.input',), - aug_op_affine, - {'rotate': Uniform(0, 360.0), - 'translate': (RandInt(-4, 4), RandInt(-4, 4)), - 'flip': (RandBool(0.5), RandBool(0.5)), - 'scale': Uniform(0.9, 1.1), - }, - {'apply': RandBool(0.5)} - ], - [ - ('data.input',), - aug_op_affine, - {'rotate': Uniform(-3.0, 3.0), - 'translate': (RandInt(-2, 2), RandInt(-2, 2)), - 'flip': (False, False), - 'scale': Uniform(0.9, 1.1), - 'channels': Choice(image_channels, probabilities=None)}, - {'apply': RandBool(0.5) if train_common_params['data.aug.phase_misalignment'] else 0} - ], - [ - ('data.input',), - unsqueeze_2d_to_3d, - {'channels': num_channels, 'axis_squeeze': 'z'}, - {} - ], - ] - augmentor = AugmentorDefault(augmentation_pipeline=aug_pipeline) - - # Create dataset - train_dataset = DatasetGenerator(cache_dest=paths['cache_dir'], - data_source=train_data_source, - processor=generate_processor, - post_processing_func=train_post_processor, - augmentor=augmentor, - statistic_keys=['data.ground_truth'] - ) - - gpu_ids_for_caching = [] - lgr.info(f'- Load and cache data:') - - train_dataset.create() - - lgr.info(f'- Load and cache data: Done') - - #### Validation data - lgr.info(f'Validation Data:', {'attrs': 'bold'}) - - ## Create data source - validation_data_source = ProstateXDataSourcePatient(paths['data_dir'], 'validation', - db_ver=DATABASE_REVISION, - db_name=train_common_params['db_name'], - fold_no=train_common_params['fold_no']) - - # post processor - validation_post_processor = partial(post_processing) - - ## Create dataset - validation_dataset = DatasetGenerator(cache_dest=paths['cache_dir'], - data_source=validation_data_source, - processor=generate_processor, - post_processing_func=validation_post_processor, - augmentor=None, - statistic_keys=['data.ground_truth'] - ) - - lgr.info(f'- Load and cache data:') - - validation_dataset.create(num_workers=0) - lgr.info(f'Data - task caching and filtering:', {'attrs': 'bold'}) - - return train_dataset, validation_dataset \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/dataset_prostate_x_folds_ver29062021_seed1.pickle b/examples/fuse_examples/imaging/classification/prostate_x/dataset_prostate_x_folds_ver29062021_seed1.pickle deleted file mode 100755 index 9c9c6c853..000000000 Binary files a/examples/fuse_examples/imaging/classification/prostate_x/dataset_prostate_x_folds_ver29062021_seed1.pickle and /dev/null differ diff --git a/examples/fuse_examples/imaging/classification/prostate_x/debug/__init__.py b/examples/fuse_examples/imaging/classification/prostate_x/debug/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/fuse_examples/imaging/classification/prostate_x/debug/check_excluded_samples_main.py b/examples/fuse_examples/imaging/classification/prostate_x/debug/check_excluded_samples_main.py new file mode 100644 index 000000000..957f381d8 --- /dev/null +++ b/examples/fuse_examples/imaging/classification/prostate_x/debug/check_excluded_samples_main.py @@ -0,0 +1,47 @@ +import numpy as np +import pandas as pd +import os +from fuseimg.datasets import prostate_x + +def main(): + data_dir = os.environ["PROSTATEX_DATA_PATH"] + all_anotations_df = get_all_annotations(data_dir) + cache_dir = "" + + + label_type = prostate_x.ProstateXLabelType.ClinSig + # bad examples: + # 'ProstateX-0025' - entirely excluded + # 'ProstateX-0005' - 2,3 [1 is used] + # 'ProstateX-0105' - 2,3 [1 is used] + # 'ProstateX-0154' - 3 [1,2, are used] + sample_ids = ['ProstateX-0005_2'] + + dataset = prostate_x.ProstateX.dataset(label_type=label_type, train=False, + cache_dir=cache_dir, data_dir=data_dir, + sample_ids=sample_ids, annotations_df=all_anotations_df) + + + + + + +def get_all_annotations(data_dir): + annotations_df = pd.read_csv(os.path.join(data_dir, 'Lesion Information', 'ProstateX-Findings-Train.csv')) + annotations_df['Patient ID'] = annotations_df['ProxID'] + annotations_df = annotations_df.set_index('ProxID') + annotations_df = annotations_df[['Patient ID', 'fid', 'ClinSig', 'pos', 'zone']] + + pids_to_fix = [('ProstateX-0159', 3), ('ProstateX-0005', 3), ('ProstateX-0025', 5)] + for pid, n_fid in pids_to_fix: + a_filter = annotations_df['Patient ID'] == pid + assert a_filter.sum() == n_fid + annotations_df.loc[a_filter, 'fid'] = np.arange(1, n_fid+1) + annotations_df['Sample ID'] = annotations_df['Patient ID']+ '_'+annotations_df['fid'].astype(str) + a_filter = annotations_df['Sample ID'].duplicated() + print(annotations_df[a_filter]['Sample ID']) + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/debug/prostatex_debug_main.py b/examples/fuse_examples/imaging/classification/prostate_x/debug/prostatex_debug_main.py new file mode 100644 index 000000000..bf2164669 --- /dev/null +++ b/examples/fuse_examples/imaging/classification/prostate_x/debug/prostatex_debug_main.py @@ -0,0 +1,223 @@ +import os + +import SimpleITK as sitk +from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe +from fuseimg.datasets import prostate_x +from fuse.data.utils.sample import create_initial_sample +from deepdiff import DeepDiff +import numpy as np +import pickle + + +import pandas as pd + +PROSTATEX_PROCESSED_FILE_DIR = '/projects/msieve_dev3/usr/Tal/prostate_x_processed_files' + + +def main(): + label_type = prostate_x.ProstateXLabelType.ClinSig + # sample_ids = prostate_x.get_samples_for_debug(n_pos=10, n_neg=10, + # label_type=prostate_x.DukeLabelType.STAGING_TUMOR_SIZE) + sample_ids = prostate_x.ProstateX.sample_ids(data_dir=os.environ["PROSTATEX_DATA_PATH"]) + sample_ids = ['ProstateX-0010_1'] # + sample_ids = ['ProstateX-0199_1'] #B-fix + sample_ids = ['ProstateX-0030_1'] + sample_ids = ['ProstateX-0159_1', 'ProstateX-0159_2', 'ProstateX-0159_3'] + + if True: + static_pipeline = prostate_x.ProstateX.static_pipeline(root_path=os.environ["PROSTATEX_DATA_PATH"], + select_series_func=prostate_x.get_selected_series_index) + + if False: + n_steps = 12 # 10 ok #9 is ok #7 is ok # 5 id ok + static_pipeline._op_ids = static_pipeline._op_ids[:n_steps] + static_pipeline._ops_and_kwargs = static_pipeline._ops_and_kwargs[:n_steps] + dynamic_pipeline = prostate_x.ProstateX.dynamic_pipeline(label_type=prostate_x.ProstateXLabelType.ClinSig, + train=True) + + print("# sample_ids=", len(sample_ids)) + for sample_id in sample_ids: + print(sample_id) + sample_dict = create_initial_sample(sample_id) + sample_dict = static_pipeline(sample_dict) + sample_dict = dynamic_pipeline(sample_dict) + print(sample_dict.flatten().keys()) + + if False: + # x_old = load_pickle('/tmp/ozery/t3.pkl') # n_steps == 9 + x_old = load_pickle('/tmp/ozery/t5.pkl') + x_new = sitk.GetArrayFromImage(sample_dict['data.input.volume4D']) + if np.all(x_old == x_new): + print( "ok") + else: + print( "not ok") + if False: # step 7 + arr_old= load_pickle('/tmp/ozery/t2.pkl') + arr_new = [sitk.GetArrayFromImage(a) for a in sample_dict['data.input.selected_volumes']] + assert len(arr_old) != len(arr_new) + for i in range(len(arr_old)): + if np.all(arr_old[i] == arr_new[i]): + print(i, "ok") + else: + print(i, "not ok") + + print("oo") + if False: # step 5 + d_old = load_pickle('/tmp/ozery/t1.pkl') + + d = {s: sample_dict[f'data.input.sequence.{s}'] for s in sample_dict['data.input.seq_ids'] } + for k, v in d.items(): + arr_new = [sitk.GetArrayFromImage(a['stk_volume']) for a in v] + arr_old = d_old[k] + if len(arr_new) != len(arr_old): + print(f"{k} mismatch in length") + else: + for ii in range(len(arr_new)): + if np.all(arr_new[ii] == arr_old[ii]): + print(f"{k} {ii} ok") + else: + print(f"{k} {ii} mismatch in content") + + print("ok") + + print("done") + if False: + # sample_ids = ['ProstateX-0058_1']#['ProstateX-0008_1'] + prostatex_dataset = prostate_x.ProstateX.dataset(data_dir=os.environ["PROSTATEX_DATA_PATH"], + label_type=prostate_x.ProstateXLabelType.ClinSig, + cache_dir=None, num_workers=0, sample_ids=sample_ids) + print("finished defining dataset, starting run") + arr = [] + rows = [] + for d in prostatex_dataset: + d2 = d.flatten() + row = (d['data.sample_id'], d['data.ground_truth']) + rows.append(row) + print("*******", row) + arr += [d2] + print(len(arr)) + print(pd.DataFrame(rows, columns=['sample_id', 'gt'])) + print(arr[0].keys()) + + +def compare_to_fuse1(): + fuse1_dir = '/tmp/ozery/prostatex_fuse1' + prostatex_dataset = prostate_x.ProstateX.dataset(data_dir=os.environ["PROSTATEX_DATA_PATH"], + label_type=prostate_x.ProstateXLabelType.ClinSig, + cache_dir=None, num_workers=16, sample_ids=None, + verbose=False) + deep_diff_config = dict(ignore_nan_inequality=True) #, math_epsilon=0.0001) + n_errors = 0 + for i, sample_dict in enumerate(prostatex_dataset): + sample_id = sample_dict['data.sample_id'] + fields = sample_id.split('_') + filename = os.path.join(fuse1_dir, f'{fields[0]}_{int(fields[1])-1}.pkl') + if not os.path.exists(filename): + print(i, f"{filename} does not exist. skipping") + n_errors += 1 + continue + d_old = load_pickle(filename) + d_new = {'data.input':sample_dict[ 'data.input.patch_volume'].numpy(), + 'data.ground_truth':sample_dict[ 'data.ground_truth']} + # diff = DeepDiff(d_old, d_new, **deep_diff_config) + s_error = '' + if d_old['data.ground_truth'] != d_new['data.ground_truth'].numpy(): + s_error += f' mismatch in label {d_old["data.ground_truth"]} in old' + if not np.all(d_old['data.input'] == d_new['data.input']): + s_error += ' mismatch in tensor' + if len(s_error)>0: + print(i, f"{filename} does not match: {s_error}") + n_errors += 1 + else: + print(i, f"{filename} ok.") + print(f"Done. Total number of mismatches={n_errors}") + + +def test_files(): + df = get_tal_prostatex_annotations_file() + print(df.shape, df['Patient ID'].nunique(), df.columns.values) + pids = set(df['Patient ID']) + df['sample_id']= df['Patient ID'] + '_'+ df['fid'].astype(str) + print(df.shape[0], df['sample_id'].nunique()) + + filenames = ['ProstateX-2-Findings-Train.csv', 'ProstateX-Findings-Train.csv'] + filenames = [os.path.join(os.environ['PROSTATEX_DATA_PATH'], 'Lesion Information', f) for f in filenames] + df_list = [] + for filename in filenames: + assert os.path.exists(filename) + df2 = pd.read_csv(filename).rename({'ProxID': 'Patient ID'}, axis=1) + + a_filter = df2['Patient ID'] == 'ProstateX-0159' + if np.any(a_filter): + assert a_filter.sum() ==3 + df2.loc[a_filter, 'fid'] = [1,2,3] + df2['sample_id'] = df2['Patient ID'] + '_' + df2['fid'].astype(str) + + df_list.append(df2) + print(df2.shape, df2.columns.values) + print(df2.iloc[0]) + print(df2.zone.unique(), df2['Patient ID'].nunique()) + df2 = df_list[1] + df3 = df_list[0] + + if False: + dd = df3.merge(df2, on=['sample_id', 'fid', 'Patient ID']) + a_filter = (dd.pos_x != dd.pos_y) | (dd.zone_x != dd.zone_y) + print(dd[a_filter]) + # change in one record: ProstateX-0005 + # values of 'ProstateX-Findings-Train.csv' matches Tal's file + + # compare df2 and df: + dd = df.merge(df2, on=['sample_id', 'fid', 'Patient ID']) + cols_2_compare = [s for s in dd.columns if s.endswith('_x')] + print("comparing to between Tal's processd file and", filenames[1]) + for sx in cols_2_compare: + sy =sx[:-1]+'y' + a_filter = dd[sx] != dd[sy] + print("checking", sx) + if np.any(a_filter): + print(dd[['sample_id', sx, sy]].loc[0]) + + print(sorted(list(set(df2['sample_id']) - set(df['sample_id'])))) + print("ok") + +def get_tal_prostatex_annotations_file(): + annotations_path = os.path.join(PROSTATEX_PROCESSED_FILE_DIR, 'dataset_prostate_x_folds_ver29062021_seed1.pickle') + with open(annotations_path, 'rb') as infile: + fold_annotations_dict = pickle.load(infile) + annotations_df = pd.concat( + [fold_annotations_dict[f'data_fold{fold}'] for fold in range(len(fold_annotations_dict))]) + + for pid, n_fid in [ ('ProstateX-0025', 5),('ProstateX-0159', 3)]: + a_filter = annotations_df['Patient ID'] == pid + if np.any(a_filter): + assert a_filter.sum() == n_fid + annotations_df.loc[a_filter, 'fid'] = np.arange(1, n_fid+1) + annotations_df['Sample ID'] = annotations_df[['Patient ID', 'fid']].apply(lambda row: '_'.join(row.values.astype(str)), axis=1) + + return annotations_df + +def compare_datasets_on_dict(): + dir1 = '/tmp/ozery/prostatex_v5' #bad + dir2 = '/tmp/ozery/prostatex_v6'# goood + deep_diff_config = dict(ignore_nan_inequality=True) # , math_epsilon=0.0001) + for i, file in enumerate(sorted(os.listdir(dir2))): + df_list = [] + for dirx in [dir1, dir2]: + filex = os.path.join(dirx, file) + # if not os.path.exists(filex): + # print(file, "does not exist") + # continue + d = load_pickle(filex) + df_list.append(d) + diff = DeepDiff(df_list[0], df_list[1], **deep_diff_config) + if len(diff)>0: + print(i, file,diff.keys()) + else: + print(i,"ok", df_list[0]['data.ground_truth'], df_list[1]['data.ground_truth']) + print("done") +if __name__ == '__main__': + # main() + # compare_to_fuse1() + test_files() + # compare_datasets_on_dict() \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/debug/prostatex_debug_main2.py b/examples/fuse_examples/imaging/classification/prostate_x/debug/prostatex_debug_main2.py new file mode 100644 index 000000000..cc061dd02 --- /dev/null +++ b/examples/fuse_examples/imaging/classification/prostate_x/debug/prostatex_debug_main2.py @@ -0,0 +1,33 @@ +import os +os.environ["PROSTATEX_DATA_PATH"] = "/projects/msieve/MedicalSieve/PatientData/ProstateX/manifest-A3Y4AE4o5818678569166032044/" + +from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe +from fuseimg.datasets import prostate_x +from fuse.data.utils.sample import create_initial_sample + +import pandas as pd + +def main(): + label_type = prostate_x.ProstateXLabelType.ClinSig + # sample_ids = prostate_x.get_samples_for_debug(n_pos=10, n_neg=10, + # label_type=prostate_x.DukeLabelType.STAGING_TUMOR_SIZE) + sample_ids = prostate_x.ProstateX.sample_ids()[:4] + prostatex_dataset = prostate_x.ProstateX.dataset(data_dir=os.environ["PROSTATEX_DATA_PATH"], + label_type=prostate_x.ProstateXLabelType.ClinSig, + cache_dir=None, num_workers=0, sample_ids=sample_ids) + print("finished defining dataset, starting run") + arr = [] + rows = [] + for d in prostatex_dataset: + d2 = d.flatten() + row = (d['data.sample_id'], d['data.ground_truth']) + rows.append(row) + print("*******", row) + arr += [d2] + print(len(arr)) + print(pd.DataFrame(rows, columns=['sample_id', 'gt'])) + print(arr[0].keys()) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/debug/test_folds_main.py b/examples/fuse_examples/imaging/classification/prostate_x/debug/test_folds_main.py new file mode 100644 index 000000000..c453bbe5b --- /dev/null +++ b/examples/fuse_examples/imaging/classification/prostate_x/debug/test_folds_main.py @@ -0,0 +1,71 @@ +import pickle +import pandas as pd +import numpy as np +from fuseimg.datasets import prostate_x +import os +from fuse_examples import fuse_examples_utils +def main(): + + df_tal_folds = read_and_test_tal_folds() + test_tals_data(df_tal_folds) + data_dir = os.environ["PROSTATEX_DATA_PATH"] + fuse_examples_dir = fuse_examples_utils.get_fuse_examples_user_dir() + cache_dir = os.path.join(fuse_examples_dir, 'prostate_x', 'cache_dir_v2') + label_type = prostate_x.ProstateXLabelType.ClinSig + dataset = prostate_x.ProstateX.dataset(label_type=label_type, train=False, + cache_dir=cache_dir, data_dir=data_dir) + + df_folds1 = df_read_and_test_folds('/projects/msieve_dev3/usr/ozery/fuse_examples/prostate_x/prostatex_8folds_v2.pkl') + +def test_tals_data(df): + n_patient_fold_pairs = (df['Patient ID']+ '_'+ df['fold'].astype(str)).nunique() + print(df.groupby(['ggg', 'is_ClinSig'])['fid'].count()) + print(df.groupby(['ClinSig', 'is_ClinSig'])['fid'].count()) + print(df.groupby(['Patient ID'])['ClinSig'].mean().unique()) + dd = df.groupby(['Patient ID'])[['fid', 'is_ClinSig']].agg({'fid': 'count', 'is_ClinSig': 'mean'}) + + dd.columns = ['#fids', '#patients'] + print(dd.groupby('#fids').count()) + + dd.columns = ['#fids', 'E(label)'] + print(dd.groupby('#fids').mean()) + print("ok") +def df_read_and_test_folds(filename): + with open(filename, 'rb') as infile: + folds_dict = pickle.load(infile) + + folds_list = [] + index_list = [] + + for fold, indexes in folds_dict.items(): + index_list += indexes + folds_list += [fold] * len(indexes) + data = np.asarray([index_list, folds_list]).T + folds_df = pd.DataFrame(data, columns=['index', 'fold']) + return folds_df +def read_and_test_tal_folds(): + filename = '/projects/msieve_dev3/usr/common/prostatex_processed_files/dataset_prostate_x_folds_ver29062021_seed1.pickle' + with open(filename, 'rb') as infile: + folds_df_map = pickle.load(infile) + + df_fold_list = [] + n_patients = 0 + for fold in range(len(folds_df_map)): + fold_name = f'data_fold{fold}' + df_fold = folds_df_map[fold_name] + df_fold['fold'] = fold + df_fold_list.append((df_fold)) + n_patients_fold=df_fold['Patient ID'].nunique() + n_patients += n_patients_fold + print(f"fold {fold}: size={df_fold.shape[0]} #patients={n_patients_fold}") + print(f"total number of (patient, fold) unique pairs={n_patients}") + df_all = pd.concat(df_fold_list, axis=0) + n_patients = df_all['Patient ID'].nunique() + print("# patients=",n_patients) + n_patient_fold_pairs = (df_all['Patient ID']+ '_'+ df_all['fold'].astype(str)).nunique() + print("# (patients, fold) pairs =", n_patient_fold_pairs) + print("ok") + return df_all + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/patient_data_source.py b/examples/fuse_examples/imaging/classification/prostate_x/patient_data_source.py deleted file mode 100644 index 15b4f2ca0..000000000 --- a/examples/fuse_examples/imaging/classification/prostate_x/patient_data_source.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -from typing import List, Tuple - -from fuse.data.data_source.data_source_base import DataSourceBase -from fuse_examples.imaging.classification.prostate_x.data_utils import ProstateXUtilsData - -class ProstateXDataSourcePatient(DataSourceBase): - def __init__(self, - db_path: str, - set_type: str, - db_name: str, - db_ver:int = 11, - fold_no: int=0, - include_gt: bool = True, - - ): - """ - Fuse DataSource for ProstateX data. - Generate sample decription per patient - :param set_type: 'train' 'validation' 'test' - :param db_ver: database version - :type include_gt: create two descriptors per patient - with 'gt' key and without - :return list of sample descriptors - """ - self.db_path = db_path - self.set_type = set_type - self.db_ver = db_ver - self.include_gt = include_gt - self.db_name = db_name - self.fold_no = fold_no - self.desc_list = self.generate_patient_list() - - - - - def get_samples_description(self): - return list(self.desc_list) - - def summary(self) -> str: - """ - See base class - """ - summary_str = '' - summary_str += f'Class = {type(self)}\n' - summary_str += f'Input source = {self.set_type}\n' - summary_str += f'Number of Patients = {len(self.desc_list)}\n' - return summary_str - - - def generate_patient_list(self) -> List[Tuple]: - ''' - Go Over all patients and create a tuple list of (db_ver, set_type, patient_id [,'gt']) - :return: list of patient descriptors - ''' - data = ProstateXUtilsData.get_dataset(self.db_path,self.set_type, self.db_ver,self.db_name,self.fold_no) - if (self.db_name=='prostate_x') | (self.db_name=='ISPY2')| (self.db_name=='DUKE'): - data_lesions = ProstateXUtilsData.get_lesions_prostate_x(data) - - patients = list(data_lesions['Patient ID'].unique()) - - return patients - -if __name__ == "__main__": - path_to_db = '/gpfs/haifa/projects/m/msieve_dev3/usr/Tal/my_research/virtual_biopsy/prostate/experiments/V1/' - train_data_source = ProstateXDataSourcePatient(path_to_db,'train',db_name='tcia', db_ver='18042021',fold_no=0, include_gt=False) \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/post_processor.py b/examples/fuse_examples/imaging/classification/prostate_x/post_processor.py deleted file mode 100644 index 57296204a..000000000 --- a/examples/fuse_examples/imaging/classification/prostate_x/post_processor.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -from typing import Dict - -import torch - -from fuse.utils.ndict import NDict -import numpy as np - - -def post_processing(batch_dict: NDict, - ) -> None: - """ - post_processing updates batch_dict on the post processing phase - :param batch_dict: - :return: - """ - # transform gt information to tensor - label_tensor = torch.tensor(batch_dict['data.ClinSig']+0,dtype=torch.int64) - batch_dict['data.ground_truth'] = label_tensor - - - - # extract zone of lesion (one of four possible zones) as possible feature to use - zone = batch_dict['data.zone'] - zone2feature = { - 'PZ': torch.tensor(np.array([0, 0, 0]), dtype=torch.float32), - 'TZ': torch.tensor(np.array([0, 0, 1]), dtype=torch.float32), - 'AS': torch.tensor(np.array([0, 1, 0]), dtype=torch.float32), - 'SV': torch.tensor(np.array([1, 0, 0]), dtype=torch.float32), - } - batch_dict['data.tensor_clinical'] = zone2feature[zone] \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/processor.py b/examples/fuse_examples/imaging/classification/prostate_x/processor.py deleted file mode 100644 index b8aeb161a..000000000 --- a/examples/fuse_examples/imaging/classification/prostate_x/processor.py +++ /dev/null @@ -1,292 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -from typing import Tuple -import os -import SimpleITK as sitk -import numpy as np -import torch -import logging -from scipy.ndimage.morphology import binary_dilation - -from fuse.data.processor.processor_base import ProcessorBase - -from fuse_examples.imaging.classification.prostate_x.data_utils import ProstateXUtilsData -from fuse.data.processor.processor_dicom_mri import DicomMRIProcessor - - -class ProstateXPatchProcessor(ProcessorBase): - """ - This processor crops the lesion volume from within 4D MRI volume base on - lesion location as appears in the database. - :returns a sample that includes: - 'patient_num': patient id - 'lesion_num': one MRI volume may include more than one lesion - 'input': vol_tensor as extracted from MRI volume processor - 'input_lesion_mask': mask_tensor, - 'ggg': row['ggg']: in prostate - lesion grade - 'zone': row['zone']: zone in prostate - 'ClinSig': row['ClinSig']: Clinical significant ( 0 for benign and 3+3 lesions, 1 for rest) - """ - def __init__(self, - vol_processor: DicomMRIProcessor = DicomMRIProcessor(), - path_to_db: str = None, - data_path: str = None, - ktrans_data_path: str = None, - db_name: str = None, - db_version: str = None, - fold_no : int = None, - lsn_shape: Tuple[int, int, int] = (16, 120, 120), - lsn_spacing: Tuple[float, float, float] = (3, 0.5, 0.5), - ): - """ - :param vol_processor - extracts 4D tensor from path to MRI dicoms - :param path_to_db: path to data pickle - :param data_path: path to directory in which dicom data is located - :param ktrans_data_path: path to directory of Ktrans seq (prostate x) - :param db_name: 'prostatex' for this example - :param fold_no: cross validation fold - :param lsn_shape: shape of volume to extract from full volume (pixels) - :param lsn_spacing: spacing of volume to extract from full volume (mm) - """ - - # store input parameters - self.vol_processor = vol_processor - self.path_to_db = path_to_db - self.data_path = data_path - self.ktrans_data_path = ktrans_data_path - self.lsn_shape = lsn_shape - self.lsn_spacing = lsn_spacing - self.db_name = db_name - self.db_ver = db_version - self.fold_no=fold_no - self.prostate_data_path = os.path.join(self.data_path,'PROSTATEx/') - - - - # ======================================================================== - def create_resample(self,vol_ref:sitk.sitkFloat32, interpolation: str, size:Tuple[int,int,int], spacing: Tuple[float,float,float]): - """ - create_resample create resample operator - :param vol_ref: sitk vol to use as a ref - :param interpolation:['linear','nn','bspline'] - :param size: in pixels () - :param spacing: in mm () - :return: resample sitk operator - """ - - if interpolation == 'linear': - interpolator = sitk.sitkLinear - elif interpolation == 'nn': - interpolator = sitk.sitkNearestNeighbor - elif interpolation == 'bspline': - interpolator = sitk.sitkBSpline - - resample = sitk.ResampleImageFilter() - resample.SetReferenceImage(vol_ref) - resample.SetOutputSpacing(spacing) - resample.SetInterpolator(interpolator) - resample.SetSize(size) - return resample - - # ======================================================================== - def apply_resampling(self,img:sitk.sitkFloat32, mask:sitk.sitkFloat32, - spacing: Tuple[float,float,float] =(0.5, 0.5, 3), size: Tuple[int,int,int] =(160, 160, 32), - transform:sitk=None, interpolation:str='bspline', - label_interpolator:sitk=sitk.sitkLabelGaussian, - ): - - ref = img if img != [] else mask - size = [int(s) for s in size] - resample = self.create_resample(ref, interpolation, size=size, spacing=spacing) - - if ~(transform is None): - resample.SetTransform(transform) - img_r = resample.Execute(img) - - resample.SetInterpolator(label_interpolator) - mask_r = resample.Execute(mask) - - - return img_r, mask_r - - # ======================================================================== - def crop_lesion_vol(self,vol:sitk.sitkFloat32, position:Tuple[float,float,float], ref:sitk.sitkFloat32, size:Tuple[int,int,int]=(160, 160, 32), - spacing:Tuple[int,int,int]=(1, 1, 3), center_slice=None): - """ - crop_lesion_vol crop tensor around position - :param vol: vol to crop - :param position: point to crop around - :param ref: reference volume - :param size: size in pixels to crop - :param spacing: spacing to resample the col - :param center_slice: z coordinates of position - :return: cropped volume - """ - - def get_lesion_mask(position, ref): - mask = np.zeros_like(sitk.GetArrayViewFromImage(ref), dtype=np.uint8) - - coords = np.round(position[::-1]).astype(np.int) - mask[coords[0], coords[1], coords[2]] = 1 - mask = binary_dilation(mask, np.ones((3, 5, 5))) + 0 - mask_sitk = sitk.GetImageFromArray(mask) - mask_sitk.CopyInformation(ref) - - return mask_sitk - - mask = get_lesion_mask(position, ref) - - vol.SetOrigin((0,) * 3) - mask.SetOrigin((0,) * 3) - vol.SetDirection(np.eye(3).flatten()) - mask.SetDirection(np.eye(3).flatten()) - - ma_centroid = mask > 0.5 - label_analysis_filer = sitk.LabelShapeStatisticsImageFilter() - label_analysis_filer.Execute(ma_centroid) - centroid = label_analysis_filer.GetCentroid(1) - offset_correction = np.array(size) * np.array(spacing)/2 - corrected_centroid = np.array(centroid) - corrected_centroid[2] = center_slice * np.array(spacing[2]) - offset = corrected_centroid - np.array(offset_correction) - - translation = sitk.TranslationTransform(3, offset) - img, mask = self.apply_resampling(vol, mask, spacing=spacing, size=size, transform=translation) - - return img, mask - - - - # ======================================================================== - def __call__(self, - sample_desc, - *args, **kwargs): - """ - Return list of samples (lesions) giving a patient level descriptor - :param sample_desc: (db_ver, set_type, patient_id) - :return: list of lesions, see TorchClassificationAlgo.create_lesion_sample() - """ - samples = [] - - # decode descriptor - patient_id= sample_desc - - # ======================================================================== - # get db - lesions - db_full = ProstateXUtilsData.get_dataset(self.path_to_db,'other',self.db_ver,self.db_name,self.fold_no) - db = ProstateXUtilsData.get_lesions_prostate_x(db_full) - - # ======================================================================== - # get patient - patient = db[db['Patient ID'] == patient_id] - # ======================================================================== - lgr = logging.getLogger('Fuse') - lgr.info(f'patient={patient_id}', {'color': 'magenta'}) - - - # ======================================================================== - # all seq paths for a certain patient - - - patient_directories = os.listdir(os.path.join(self.prostate_data_path, patient_id)) - patient_directories = patient_directories[0] - images_path = os.path.join(self.prostate_data_path, patient_id, patient_directories) - - # ======================================================================== - # vol_4D is multichannel volume (z,x,y,chan(sequence)) - vol_4D,vol_ref = self.vol_processor((images_path,self.ktrans_data_path,patient_id)) - - # ======================================================================== - # each row contains one lesion, iterate over lesions - - for index, row in patient.iterrows(): - #read original position - pos_orig = np.array(np.fromstring(row.values[1], dtype=np.float32, sep=' ')) - # transform to pixel coordinate in ref coords - pos_vol = np.array(vol_ref.TransformPhysicalPointToContinuousIndex(pos_orig.astype(np.float64))) - # crop lesion vol - vol_cropped, mask_cropped = self.crop_lesion_vol( - vol_4D, pos_vol,vol_ref ,center_slice=pos_vol[2], - size=(self.lsn_shape[2], self.lsn_shape[1], self.lsn_shape[0]), - spacing=(self.lsn_spacing[2], self.lsn_spacing[1], self.lsn_spacing[0])) - - vol_cropped_tmp = sitk.GetArrayFromImage(vol_cropped) - if len(vol_cropped_tmp.shape)<4: - # fix dimensions in case of one seq - vol_cropped_tmp = vol_cropped_tmp[:,:,:,np.newaxis] - vol = np.moveaxis(vol_cropped_tmp, 3, 0) - else: - vol = np.moveaxis(sitk.GetArrayFromImage(vol_cropped), 3, 0) - - if np.isnan(vol).any(): - input[np.isnan(input)] = 0 - - mask = sitk.GetArrayFromImage(mask_cropped) - vol_tensor = torch.from_numpy(vol).type(torch.FloatTensor) - mask_tensor = torch.from_numpy(mask).unsqueeze(0).type(torch.FloatTensor) - - # sample - sample = { - 'patient_num': patient_id, - 'lesion_num': row['fid'], - 'input': vol_tensor, - 'input_lesion_mask': mask_tensor, - 'ggg': row['ggg'], - 'zone': row['zone'], - 'ClinSig': row['ClinSig'], - - } - - samples.append(sample) - - - - return samples - - -if __name__ == "__main__": - import matplotlib.pyplot as plt - import pandas as pd - - path_to_db = '/gpfs/haifa/projects/m/msieve_dev3/usr/Tal/my_research/virtual_biopsy/prostate/experiments/V4/' - dataset = 'prostate_x' - if dataset=='prostate_x': - # for ProstateX - path_to_dataset = '/projects/msieve/MedicalSieve/PatientData/ProstateX/manifest-A3Y4AE4o5818678569166032044/' - prostate_data_path = path_to_dataset - Ktrain_data_path = path_to_dataset + '/ProstateXKtrains-train-fixed/' - sample = ('29062021', 'train', 'ProstateX-0148', 'pred') - - a = ProstateXPatchProcessor(vol_processor=DicomMRIProcessor(reference_inx=0),path_to_db = path_to_db, - data_path=prostate_data_path,ktrans_data_path=Ktrain_data_path, - db_name=dataset,fold_no=1,lsn_shape=(13, 74, 74)) - samples = a.__call__(sample) - l_seq = pd.read_csv('/gpfs/haifa/projects/m/msieve_dev3/usr/Tal/my_research/virtual_biopsy/prostate/prostate_x/metadata.csv') - for sample_id in list(l_seq['Subject ID'].unique()): - # sample_id = 'ACRIN-6698-760011' - sample = ('29062021', 'validation', sample_id, 'pred') - samples = a.__call__(sample) - if len(samples)==0: - sample = ('29062021', 'train', sample_id, 'pred') - samples = a.__call__(sample) - - path2save = '/gpfs/haifa/projects/m/msieve_dev3/usr/Tal/my_research/virtual_biopsy/prostate/prostate_x/data_visualization/' - fix, ax = plt.subplots(nrows=5, ncols=13, sharex=True, sharey=True) - for idx in range(5): - for jdx in range(13): - ll = samples[0]['input'].cpu().detach().numpy()[idx, jdx, :, :] - ax[idx, jdx].imshow(ll, cmap='gray') - fix.suptitle(sample_id) - fix.savefig(path2save + sample_id + '.jpg') \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/run_train_3dpatch.py b/examples/fuse_examples/imaging/classification/prostate_x/run_train_3dpatch.py deleted file mode 100644 index cff2aec5a..000000000 --- a/examples/fuse_examples/imaging/classification/prostate_x/run_train_3dpatch.py +++ /dev/null @@ -1,362 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -import logging -import os -import pathlib -import torch.nn.functional as F -import torch.optim as optim -from torch.utils.data.dataloader import DataLoader -from fuse.eval.metrics.classification.metrics_classification_common import MetricAUCROC, MetricROCCurve -from fuse.eval.evaluator import EvaluatorDefault - -from fuse.data.dataset.dataset_base import DatasetBase -from fuse.data.sampler.sampler_balanced_batch import SamplerBalancedBatch - -from fuse.dl.losses.loss_default import LossDefault -from fuse.dl.managers.callbacks.callback_metric_statistics import MetricStatisticsCallback -from fuse.dl.managers.callbacks.callback_tensorboard import TensorboardCallback -from fuse.dl.managers.callbacks.callback_time_statistics import TimeStatisticsCallback -from fuse.dl.managers.manager_default import ManagerDefault - -import fuse.utils.gpu as GPU -from fuse.utils.utils_logger import fuse_logger_start - - -from fuse_examples.imaging.classification.prostate_x.dataset import prostate_x_dataset -from fuse_examples.imaging.classification.prostate_x.backbone_3d_multichannel import Fuse_model_3d_multichannel,ResNet -from fuse_examples.imaging.classification.prostate_x.patient_data_source import ProstateXDataSourcePatient -from fuse_examples.imaging.classification.prostate_x.tasks import ProstateXTask -from fuse.dl.models.heads import Head1dClassifier - - -########################################## -# Output Paths -# ########################################## - -# TODO: path to save model -root_path ='.' -# TODO: path for prostateX data -# Download instructions can be found in README -# load data from: -# https://wiki.cancerimagingarchive.net/display/Public/SPIE-AAPM-NCI+PROSTATEx+Challenges#23691656d4622c5ad5884bdb876d6d441994da38 -root_data = 'PatientData/ProstateX/manifest-A3Y4AE4o5818678569166032044/' - - -PATHS = {'force_reset_model_dir': False, - # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. - 'model_dir': os.path.join(root_path, 'prostatex/my_model/'), - 'cache_dir': os.path.join(root_path, 'prostatex/my_cache/'), - 'inference_dir': os.path.join(root_path, 'prostatex/my_model/inference/'), - 'eval_dir': os.path.join(root_path, 'prostatex/my_model/eval/'), - 'data_dir': pathlib.Path(__file__).parent.resolve(), - 'prostate_data_path' : root_data, - 'ktrans_path': os.path.join(root_data, 'ProstateXKtrains-train-fixed/'), - } -################################# -# Train Template -################################# -########################################## -# Train Common Params -########################################## -# ============ -# Data -# ============ -TRAIN_COMMON_PARAMS = {} -TRAIN_COMMON_PARAMS['db_name'] = 'prostate_x' -TRAIN_COMMON_PARAMS['db_version'] = 29062021 -TRAIN_COMMON_PARAMS['fold_no'] = 5 -TRAIN_COMMON_PARAMS['data.batch_size'] = 50 -TRAIN_COMMON_PARAMS['data.train_num_workers'] = 8 -TRAIN_COMMON_PARAMS['data.validation_num_workers'] = 8 -# add misalignment to segmentation -TRAIN_COMMON_PARAMS['data.aug.mask_misalignment'] = True -# add misalignment to phase registration -TRAIN_COMMON_PARAMS['data.aug.phase_misalignment'] = True - -# =============== -# Manager - Train -# =============== -TRAIN_COMMON_PARAMS['manager.train_params'] = { - 'num_gpus': 1, - 'num_epochs': 5, - 'virtual_batch_size': 1, # number of batches in one virtual batch - 'start_saving_epochs': 120, # first epoch to start saving checkpoints from - 'gap_between_saving_epochs': 5, # number of epochs between saved checkpoint -} -TRAIN_COMMON_PARAMS['manager.best_epoch_source'] = [ - { - 'source': 'metrics.auc.macro_avg', # can be any key from losses or metrics dictionaries - 'optimization': 'max', # can be either min/max - 'on_equal_values': 'better', - # can be either better/worse - whether to consider best epoch when values are equal - }, - -] -TRAIN_COMMON_PARAMS['manager.learning_rate'] = 1e-5 -TRAIN_COMMON_PARAMS['manager.weight_decay'] = 1e-4 -TRAIN_COMMON_PARAMS['manager.dropout'] = 0.5 -TRAIN_COMMON_PARAMS['manager.momentum'] = 0.9 -TRAIN_COMMON_PARAMS['manager.resume_checkpoint_filename'] = None # if not None, will try to load the checkpoint - -TRAIN_COMMON_PARAMS['num_backbone_features'] = 512 -TRAIN_COMMON_PARAMS['task'] = ProstateXTask('ClinSig', 0) -TRAIN_COMMON_PARAMS['class_num'] = TRAIN_COMMON_PARAMS['task'].num_classes() - -# backbone parameters -TRAIN_COMMON_PARAMS['backbone_model_dict'] = \ - {'input_channels_num': 5, - } - -def train_template(paths: dict, train_common_params: dict): - # ============================================================================== - # Logger - # ============================================================================== - fuse_logger_start(output_path=paths['model_dir'], console_verbose_level=logging.INFO, - list_of_source_files=[]) - lgr = logging.getLogger('Fuse') - lgr.info('Fuse Train', {'attrs': ['bold', 'underline']}) - - lgr.info(f'model_dir={paths["model_dir"]}', {'color': 'magenta'}) - lgr.info(f'cache_dir={paths["cache_dir"]}', {'color': 'magenta'}) - - # ============================================================================== - # Data - # ============================================================================== - train_dataset, validation_dataset = prostate_x_dataset(paths,train_common_params,lgr) - - ## Create dataloader - lgr.info(f'- Create sampler:') - - sampler = SamplerBalancedBatch(dataset=train_dataset, - balanced_class_name='data.ground_truth', - num_balanced_classes=train_common_params['task'].num_classes(), - batch_size=train_common_params['data.batch_size'], - balanced_class_weights= - [int(train_common_params['data.batch_size']/train_common_params['class_num'])] * train_common_params['class_num'], - use_dataset_cache=True) - - lgr.info(f'- Create sampler: Done') - - # ## Create dataloader - train_dataloader = DataLoader(dataset=train_dataset, - batch_sampler=sampler, - collate_fn=train_dataset.collate_fn, - num_workers=train_common_params['data.train_num_workers']) - lgr.info(f'Train Data: Done', {'attrs': 'bold'}) - - validation_dataloader = DataLoader(dataset=validation_dataset, - shuffle=False, - drop_last=False, - batch_size=train_common_params['data.batch_size'], - num_workers=train_common_params['data.validation_num_workers'], - collate_fn=validation_dataset.collate_fn) - lgr.info(f'Validation Data: Done', {'attrs': 'bold'}) - - # ============================================================================== - # Model - # ============================================================================== - lgr.info('Model:', {'attrs': 'bold'}) - - model = Fuse_model_3d_multichannel( - conv_inputs=(('data.input', 1),), - backbone= ResNet(ch_num=TRAIN_COMMON_PARAMS['backbone_model_dict']['input_channels_num']), - heads=[ - Head1DClassifier(head_name='ClinSig', - conv_inputs=[('model.backbone_features', train_common_params['num_backbone_features'])], - post_concat_inputs=None, - dropout_rate=0.25, - shared_classifier_head=None, - layers_description=None, - num_classes=2), - - ] - ) - lgr.info('Model: Done', {'attrs': 'bold'}) - - # ==================================================================================== - # Loss - # ==================================================================================== - lgr.info('Losses: CrossEntropy', {'attrs': 'bold'}) - - losses = { - 'cls_loss': LossDefault(pred='model.logits.ClinSig', - target='data.ground_truth', - callable=F.cross_entropy, weight=1.0), - } - - - # ==================================================================================== - # Metrics - # ==================================================================================== - lgr.info('Metrics:', {'attrs': 'bold'}) - - metrics = { - - 'auc': MetricAUCROC(pred='model.output.ClinSig', target='data.ground_truth', - class_names=train_common_params['task'].class_names()), - } - - - # ===================================================================================== - # Callbacks - # ===================================================================================== - callbacks = [ - TensorboardCallback(model_dir=paths['model_dir']), # save statistics for tensorboard - MetricStatisticsCallback(output_path=paths['model_dir'] + "/metrics.csv"), - # save statistics for tensorboard in a csv file - TimeStatisticsCallback(num_epochs=train_common_params['manager.train_params']['num_epochs'], - load_expected_part=0.1) # time profiler - ] - - # ===================================================================================== - # Manager - Train - # ===================================================================================== - lgr.info('Train:', {'attrs': 'bold'}) - - # create optimizer - optimizer = optim.Adam(model.parameters(), - lr=train_common_params['manager.learning_rate'], - weight_decay=train_common_params['manager.weight_decay']) - - # create scheduler - scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True) - - # train from scratch - manager = ManagerDefault(output_model_dir=paths['model_dir'], force_reset=paths['force_reset_model_dir']) - # Providing the objects required for the training process. - manager.set_objects(net=model, - optimizer=optimizer, - losses=losses, - metrics=metrics, - best_epoch_source=train_common_params['manager.best_epoch_source'], - lr_scheduler=scheduler, - callbacks=callbacks, - train_params=train_common_params['manager.train_params']) - - ## Continue training - if train_common_params['manager.resume_checkpoint_filename'] is not None: - # Loading the checkpoint including model weights, learning rate, and epoch_index. - manager.load_checkpoint(checkpoint=train_common_params['manager.resume_checkpoint_filename'], mode='train') - - # Start training - manager.train(train_dataloader=train_dataloader, - validation_dataloader=validation_dataloader) - - lgr.info('Train: Done', {'attrs': 'bold'}) - - - -###################################### -# Inference Common Params -###################################### -INFER_COMMON_PARAMS = {} -INFER_COMMON_PARAMS['db_version'] = 29062021 -INFER_COMMON_PARAMS['db_name'] = 'prostate_x' -INFER_COMMON_PARAMS['fold_no'] = 5 -INFER_COMMON_PARAMS['infer_filename'] = os.path.join(PATHS['inference_dir'], 'validation_set_infer.pickle.gz') -INFER_COMMON_PARAMS['checkpoint'] = 'best' # Fuse TIP: possible values are 'best', 'last' or epoch_index. - -###################################### -# Inference Template -###################################### -def infer_template(paths: dict, infer_common_params: dict): - #### Logger - # fuse_logger_start(output_path=paths['inference_dir'], console_verbose_level=logging.INFO) - lgr = logging.getLogger('Fuse') - lgr.info('Fuse Inference', {'attrs': ['bold', 'underline']}) - lgr.info(f'infer_filename={infer_common_params["infer_filename"]}', {'color': 'magenta'}) - - #### create infer data set - - lgr.info(f'db_name={infer_common_params["db_name"]}', {'color': 'magenta'}) - ## Create data source: - infer_data_source = ProstateXDataSourcePatient(paths['data_dir'],'validation', - db_ver=infer_common_params['db_version'], - db_name = infer_common_params['db_name'], - fold_no=infer_common_params['fold_no']) - - ### load dataset - data_set_filename = os.path.join(paths["model_dir"], "inference_dataset.pth") - dataset = DatasetBase.load(filename=data_set_filename, override_datasource=infer_data_source, override_cache_dest=paths["cache_dir"], num_workers=0) - dataloader = DataLoader(dataset=dataset, - shuffle=False, - drop_last=False, - batch_size=50, - num_workers=5, - collate_fn=dataset.collate_fn) - #### Manager for inference - manager = ManagerDefault() - # extract just the global classification per sample and save to a file - output_columns = ['model.output.ClinSig','data.ground_truth'] - manager.infer(data_loader=dataloader, - input_model_dir=paths['model_dir'], - checkpoint=infer_common_params['checkpoint'], - output_columns=output_columns, - output_file_name=infer_common_params['infer_filename']) - -###################################### -# Analyze Common Params -###################################### -EVAL_COMMON_PARAMS = {} -EVAL_COMMON_PARAMS['infer_filename'] = INFER_COMMON_PARAMS['infer_filename'] -###################################### -# Analyze Template -###################################### -def eval_template(paths: dict, eval_common_params: dict): - fuse_logger_start(output_path=None, console_verbose_level=logging.INFO) - lgr = logging.getLogger('Fuse') - lgr.info('Fuse Eval', {'attrs': ['bold', 'underline']}) - - - # metrics - metrics = { - 'roc': MetricROCCurve(pred='model.output.ClinSig', target='data.ground_truth', - output_filename=os.path.join(paths['inference_dir'], 'roc_curve.png')), - 'auc': MetricAUCROC(pred='model.output.ClinSig', target='data.ground_truth') - } - - # create evaluator - evaluator = EvaluatorDefault() - - # run - results = evaluator.eval(ids=None, - data=eval_common_params["infer_filename"], - metrics=metrics, - output_dir=paths["eval_dir"]) - - return results -###################################### -# Run -###################################### -if __name__ == "__main__": - - # allocate gpus - NUM_GPUS = 1 - if NUM_GPUS == 0: - TRAIN_COMMON_PARAMS['manager.train_params']['device'] = 'cpu' - # uncomment if you want to use specific gpus instead of automatically looking for free ones - force_gpus = None # [0] - GPU.choose_and_enable_multiple_gpus(NUM_GPUS, force_gpus=force_gpus) - - RUNNING_MODES = ['train','infer', 'eval'] # Options: 'train', 'infer', 'eval' - - if 'train' in RUNNING_MODES: - train_template(paths=PATHS, train_common_params=TRAIN_COMMON_PARAMS) - - if 'infer' in RUNNING_MODES: - infer_template(paths=PATHS, infer_common_params=INFER_COMMON_PARAMS) - - if 'eval' in RUNNING_MODES: - eval_template(paths=PATHS,eval_common_params=EVAL_COMMON_PARAMS) \ No newline at end of file diff --git a/examples/fuse_examples/imaging/classification/prostate_x/runner_prostate_x.py b/examples/fuse_examples/imaging/classification/prostate_x/runner_prostate_x.py new file mode 100644 index 000000000..9d9d7e40d --- /dev/null +++ b/examples/fuse_examples/imaging/classification/prostate_x/runner_prostate_x.py @@ -0,0 +1,530 @@ +""" +(C) Copyright 2021 IBM Corp. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Created on June 30, 2021 + +""" + +import torch +import logging +import os +import copy +from typing import OrderedDict, Optional + +import torch.nn.functional as F +import torch.optim as optim +from torch.utils.data.dataloader import DataLoader + +import fuse.utils.gpu as GPU +from fuse_examples.fuse_examples_utils import ask_user +from fuse_examples.imaging.utils.backbone_3d_multichannel import Fuse_model_3d_multichannel, ResNet +from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.samplers import BatchSamplerDefault +from fuse.data.utils.split import dataset_balanced_division_to_folds +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.models.heads import Head1DClassifier +from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.classification.metrics_classification_common import MetricAccuracy, MetricAUCROC, MetricROCCurve +from fuse.eval.metrics.classification.metrics_thresholding_common import MetricApplyThresholds +from fuse.utils.rand.seed import Seed +from fuse.utils.utils_debug import FuseDebug +from fuse.utils.utils_logger import fuse_logger_start +from fuse.utils.file_io.file_io import create_dir, load_pickle, save_dataframe + +from fuseimg.datasets import prostate_x + +from fuse.dl.lightning.pl_module import LightningModuleDefault +from fuse.dl.lightning.pl_funcs import convert_predictions_to_dataframe +import pytorch_lightning as pl +from fuse_examples import fuse_examples_utils + +from fuse.data.utils.export import ExportDataset + + +def main(): + mode = 'default' # Options: 'default', 'fast', 'debug', 'verbose', 'user'. See details in FuseDebug + + # allocate gpus + # To use cpu - set NUM_GPUS to 0 + if mode == 'debug': + NUM_GPUS = 1 + else: + NUM_GPUS = 1 + # uncomment if you want to use specific gpus instead of automatically looking for free ones + force_gpus = None # [0] + GPU.choose_and_enable_multiple_gpus(NUM_GPUS, force_gpus=force_gpus) + + PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS = get_setting(mode, num_devices=NUM_GPUS, n_folds=8, + heldout_fold=4) + print(PATHS) + + RUNNING_MODES = ['train', 'infer', 'eval'] # Options: 'train', 'infer', 'eval' + # train + if 'train' in RUNNING_MODES: + print(TRAIN_COMMON_PARAMS) + run_train(paths=PATHS, train_params=TRAIN_COMMON_PARAMS) + + # infer + if 'infer' in RUNNING_MODES: + print(INFER_COMMON_PARAMS) + run_infer(paths=PATHS, infer_common_params=INFER_COMMON_PARAMS, audit_cache='train' not in RUNNING_MODES) + + # eval + if 'eval' in RUNNING_MODES: + print(EVAL_COMMON_PARAMS) + run_eval(paths=PATHS, eval_common_params=EVAL_COMMON_PARAMS) + + print(f"Done running with heldout={INFER_COMMON_PARAMS['data.infer_folds']}") + + +def get_setting(mode, num_devices, label_type=prostate_x.ProstateXLabelType.ClinSig, n_folds=8, heldout_fold=7): + ########################################################################################################### + # Fuse + ########################################################################################################### + ########################################## + # Debug modes + ########################################## + + debug = FuseDebug(mode) + input_channels_num = 5 + ########################################## + # Output Paths + ########################################## + assert "PROSTATEX_DATA_PATH" in os.environ, "Expecting environment variable PROSTATEX_DATA_PATH to be set. Follow the instruction in example README file to download and set the path to the data" + data_dir = os.environ["PROSTATEX_DATA_PATH"] + ROOT = "./_examples/prostate_x" + + if mode == 'debug': + data_split_file = os.path.join(ROOT, f'prostate_x_{n_folds}_folds_debug.pkl') + selected_sample_ids = prostate_x.get_samples_for_debug(n_pos=10, n_neg=10, label_type=label_type) + print(selected_sample_ids) + cache_dir = os.path.join(ROOT, 'cache_dir_debug') + model_dir = os.path.join(ROOT, 'model_dir_debug') + num_workers = 0 + batch_size = 2 + num_epoch = 5 + else: + data_split_file = os.path.join(ROOT, f'prostatex_{n_folds}_folds.pkl') + cache_dir = os.path.join(ROOT, f'cache_dir_pl') + model_dir = os.path.join(ROOT, f'model_dir_pl_{heldout_fold}') + selected_sample_ids = None + + num_workers = 16 + batch_size = 50 + num_epoch = 50 + PATHS = {'model_dir': model_dir, + 'cache_dir': cache_dir, + 'data_split_filename': data_split_file, + 'data_dir': data_dir, + 'inference_dir': os.path.join(model_dir, 'infer_dir'), + 'eval_dir': os.path.join(model_dir, 'eval_dir'), + } + + ########################################## + # Train Common Params + ########################################## + TRAIN_COMMON_PARAMS = {} + + # ============ + # Data + # ============ + + train_folds = [i % n_folds for i in range(heldout_fold + 1, heldout_fold + n_folds - 1)] + validation_fold = (heldout_fold - 1) % n_folds + TRAIN_COMMON_PARAMS['data.selected_sample_ids'] = selected_sample_ids + TRAIN_COMMON_PARAMS['data.batch_size'] = batch_size + TRAIN_COMMON_PARAMS['data.train_num_workers'] = num_workers + TRAIN_COMMON_PARAMS['data.validation_num_workers'] = num_workers + TRAIN_COMMON_PARAMS['data.num_folds'] = n_folds + TRAIN_COMMON_PARAMS['data.train_folds'] = train_folds + TRAIN_COMMON_PARAMS['data.validation_folds'] = [validation_fold] + + # =============== + # PL Trainer + # =============== + TRAIN_COMMON_PARAMS['trainer.num_epochs'] = num_epoch + TRAIN_COMMON_PARAMS['trainer.num_devices'] = num_devices + TRAIN_COMMON_PARAMS['trainer.accelerator'] = "gpu" + TRAIN_COMMON_PARAMS['trainer.ckpt_path'] = None # path to the checkpoint you wish continue the training from + + # =============== + # Optimizer + # =============== + TRAIN_COMMON_PARAMS['opt.lr'] = 1e-3 + TRAIN_COMMON_PARAMS['opt.weight_decay'] = 0.005 + + # =============== + # Manager - Train + # =============== + TRAIN_COMMON_PARAMS['manager.train_params'] = { + 'num_epochs': num_epoch, + 'virtual_batch_size': 1, # number of batches in one virtual batch + 'start_saving_epochs': 10, # first epoch to start saving checkpoints from + 'gap_between_saving_epochs': 5, # number of epochs between saved checkpoint + } + TRAIN_COMMON_PARAMS['manager.best_epoch_source'] = { + 'source': 'metrics.auc', # can be any key from 'epoch_results' + 'optimization': 'max', # can be either min/max + 'on_equal_values': 'better', + # can be either better/worse - whether to consider best epoch when values are equal + } + TRAIN_COMMON_PARAMS['manager.learning_rate'] = 1e-5 + TRAIN_COMMON_PARAMS['manager.weight_decay'] = 1e-4 + TRAIN_COMMON_PARAMS['manager.dropout'] = 0.5 + TRAIN_COMMON_PARAMS['manager.momentum'] = 0.9 + TRAIN_COMMON_PARAMS['manager.resume_checkpoint_filename'] = None # if not None, will try to load the checkpoint + # TRAIN_COMMON_PARAMS['imaging_dropout'] = 0.25 + # # TRAIN_COMMON_PARAMS['fused_dropout'] = 0.0 + # # TRAIN_COMMON_PARAMS['clinical_dropout'] = 0.0 + + TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] = 512 + + # in order to add relevant tabular feature uncomment: + # num_backbone_features_clinical, post_concat_inputs,post_concat_model + TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] = None # 256 + TRAIN_COMMON_PARAMS['post_concat_inputs'] = None # [('data.clinical_features',9),] + TRAIN_COMMON_PARAMS['post_concat_model'] = None # (256,256) + + if TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] is None: + TRAIN_COMMON_PARAMS['num_backbone_features'] = TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] + else: + TRAIN_COMMON_PARAMS['num_backbone_features'] = \ + TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] + TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] + + # classification task: + # supported tasks are: 'ClinSig' + TRAIN_COMMON_PARAMS['label_type'] = label_type + TRAIN_COMMON_PARAMS['class_num'] = label_type.get_num_classes() + + # backbone parameters + TRAIN_COMMON_PARAMS['backbone_model_dict'] = \ + {'input_channels_num': input_channels_num, + } + + # ============ + # Model + # ============ + + TRAIN_COMMON_PARAMS['model'] = dict(imaging_dropout=0.25, + # fused_dropout=0.0, + # clinical_dropout=0.0, + num_backbone_features=TRAIN_COMMON_PARAMS['num_backbone_features'], + input_channels_num=5, + ) + + ###################################### + # Inference Common Params + ###################################### + INFER_COMMON_PARAMS = {} + INFER_COMMON_PARAMS['infer_filename'] = 'infer_file.gz' + INFER_COMMON_PARAMS['checkpoint'] = "best_epoch.ckpt" + INFER_COMMON_PARAMS['data.infer_folds'] = [heldout_fold] # infer validation set + INFER_COMMON_PARAMS['data.batch_size'] = 4 + INFER_COMMON_PARAMS['data.num_workers'] = num_workers + INFER_COMMON_PARAMS['label_type'] = TRAIN_COMMON_PARAMS['label_type'] + INFER_COMMON_PARAMS['model'] = TRAIN_COMMON_PARAMS['model'] + INFER_COMMON_PARAMS['trainer.num_devices'] = num_devices + INFER_COMMON_PARAMS['trainer.accelerator'] = "gpu" + + ###################################### + # Analyze Common Params + ###################################### + EVAL_COMMON_PARAMS = {} + EVAL_COMMON_PARAMS['infer_filename'] = INFER_COMMON_PARAMS['infer_filename'] + + return PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS + + +def create_model(imaging_dropout: float, num_backbone_features: int, input_channels_num: int) -> torch.nn.Module: + """ + creates the model + See Head3DClassifier for details about imaging_dropout, clinical_dropout, fused_dropout + """ + conv_inputs = (('data.input.patch_volume', 1),) + + model = Fuse_model_3d_multichannel( + conv_inputs=conv_inputs, # previously 'data.input'. could be either 'data.input.patch_volume' or 'data.input.patch_volume_orig' + backbone=ResNet(conv_inputs=conv_inputs, ch_num=input_channels_num), + # since backbone resnet contains pooling and fc, the feature output is 1D, + # hence we use Head1dClassifier as classification head + heads=[ + Head1DClassifier(head_name='classification', + conv_inputs=[('model.backbone_features', num_backbone_features)], + post_concat_inputs=None, # [('data.clinical_features',9),] + post_concat_model=None, # (256,256) + dropout_rate=imaging_dropout, + # append_dropout_rate=train_params['clinical_dropout'], + # fused_dropout_rate=train_params['fused_dropout'], + shared_classifier_head=None, + layers_description=None, + num_classes=2, + # append_features=[("data.input.clinical", 8)], + # append_layers_description=(256,128), + ), + ] + ) + return model + + +################################# +# Train Template +################################# +def run_train(paths: dict, train_params: dict): + Seed.set_seed(222, False) + + # ============================================================================== + # Logger + # ============================================================================== + fuse_logger_start(output_path=paths['model_dir'], console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Train', {'attrs': ['bold', 'underline']}) + + lgr.info(f'model_dir={paths["model_dir"]}', {'color': 'magenta'}) + lgr.info(f'cache_dir={paths["cache_dir"]}', {'color': 'magenta'}) + lgr.info(f'train folds={train_params["data.train_folds"]}', {'color': 'magenta'}) + lgr.info(f'validation folds={train_params["data.validation_folds"]}', {'color': 'magenta'}) + + # ============================================================================== + # Data + # ============================================================================== + # Train Data + lgr.info(f'Train Data:', {'attrs': 'bold'}) + + reset_cache = ask_user('Do you want to reset cache?') + cache_kwargs = {'use_pipeline_hash': False} + if not reset_cache: + audit_cache = ask_user('Do you want to audit cache?') + if not audit_cache: + cache_kwargs2 = dict(audit_first_sample=False, audit_rate=None) + cache_kwargs = {**cache_kwargs, **cache_kwargs2} + + # split to folds randomly + params = dict(label_type=train_params['label_type'], data_dir=paths["data_dir"], cache_dir=paths["cache_dir"], + reset_cache=reset_cache, sample_ids=train_params['data.selected_sample_ids'], + num_workers=train_params['data.train_num_workers'], + cache_kwargs=cache_kwargs, train=False, verbose=False) + + dataset_all = prostate_x.ProstateX.dataset(**params) + # ExportDataset.export_to_dir(dataset=dataset_all, output_dir=f'/tmp/ozery/prostatex_{my_version}') + + folds = dataset_balanced_division_to_folds(dataset=dataset_all, + output_split_filename=paths["data_split_filename"], + keys_to_balance=["data.ground_truth"], + id='data.input.patient_id', + workers=0, # todo: stuck in Export to dataframe + nfolds=train_params["data.num_folds"], + verbose=True, + reset_split=True) + + train_sample_ids = [] + for fold in train_params["data.train_folds"]: + train_sample_ids += folds[fold] + validation_sample_ids = [] + for fold in train_params["data.validation_folds"]: + validation_sample_ids += folds[fold] + + params['sample_ids'] = train_sample_ids + params['reset_cache'] = False + params['train'] = True + params['cache_kwargs'] = dict(use_pipeline_hash=False, audit_first_sample=False, audit_rate=None) + train_dataset = prostate_x.ProstateX.dataset(**params) + # for _ in train_dataset: + # pass + params['sample_ids'] = validation_sample_ids + params['train'] = False + validation_dataset = prostate_x.ProstateX.dataset(**params) + + lgr.info(f'- Create sampler:') + sampler = BatchSamplerDefault(dataset=train_dataset, + balanced_class_name='data.ground_truth', + num_balanced_classes=train_params['class_num'], + batch_size=train_params['data.batch_size'], + workers=0 # train_params['data.train_num_workers'] #todo: stuck + ) + lgr.info(f'- Create sampler: Done') + + # Create dataloader + train_dataloader = DataLoader(dataset=train_dataset, + batch_sampler=sampler, + collate_fn=CollateDefault(), + num_workers=train_params['data.train_num_workers']) + lgr.info(f'Train Data: Done', {'attrs': 'bold'}) + + # dataloader + validation_dataloader = DataLoader(dataset=validation_dataset, + batch_size=train_params['data.batch_size'], + collate_fn=CollateDefault(), + num_workers=train_params['data.validation_num_workers']) + lgr.info(f'Validation Data: Done', {'attrs': 'bold'}) + + # ============================================================================== + # Model + # ============================================================================== + lgr.info('Model:', {'attrs': 'bold'}) + + model = create_model(**train_params["model"]) + + lgr.info('Model: Done', {'attrs': 'bold'}) + + # ==================================================================================== + # Loss + # ==================================================================================== + losses = { + 'cls_loss': LossDefault(pred='model.logits.classification', + target='data.ground_truth', callable=F.cross_entropy, weight=1.0), + } + + # ==================================================================================== + # Metrics + # ==================================================================================== + lgr.info('Metrics:', {'attrs': 'bold'}) + train_metrics = OrderedDict([ + ('auc', MetricAUCROC(pred='model.output.classification', target='data.ground_truth')) + ]) + validation_metrics = copy.deepcopy(train_metrics) # use the same metrics in validation as well + + # either a dict with arguments to pass to ModelCheckpoint or list dicts for multiple ModelCheckpoint callbacks (to monitor and save checkpoints for more then one metric). + best_epoch_source = dict( + monitor="validation.metrics.auc", + mode="max", + ) + + # create optimizer + optimizer = optim.SGD(model.parameters(), lr=train_params['opt.lr'], weight_decay=train_params['opt.weight_decay'], momentum=0.9, nesterov=True) + + # create learning scheduler + lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer) + lr_sch_config = dict(scheduler=lr_scheduler, + monitor="validation.losses.total_loss") + + # optimizier and lr sch - see pl.LightningModule.configure_optimizers return value for all options + optimizers_and_lr_schs = dict(optimizer=optimizer, lr_scheduler=lr_sch_config) + + # ===================================================================================== + # Train + # ===================================================================================== + lgr.info('Train:', {'attrs': 'bold'}) + + # create instance of PL module - FuseMedML generic version + pl_module = LightningModuleDefault(model_dir=paths["model_dir"], + model=model, + losses=losses, + train_metrics=train_metrics, + validation_metrics=validation_metrics, + best_epoch_source=best_epoch_source, + optimizers_and_lr_schs=optimizers_and_lr_schs) + + # create lightining trainer. + pl_trainer = pl.Trainer(default_root_dir=paths['model_dir'], + max_epochs=train_params['trainer.num_epochs'], + accelerator=train_params["trainer.accelerator"], + devices=train_params["trainer.num_devices"], + auto_select_gpus=True) + + # train + pl_trainer.fit(pl_module, train_dataloader, validation_dataloader, ckpt_path=train_params['trainer.ckpt_path']) + + lgr.info('Train: Done', {'attrs': 'bold'}) + + +###################################### +# Inference Template +###################################### +def run_infer(paths: dict, infer_common_params: dict, audit_cache: Optional[bool] = True): + create_dir(paths['inference_dir']) + infer_file = os.path.join(paths['inference_dir'], infer_common_params['infer_filename']) + checkpoint_file = os.path.join(paths['model_dir'], infer_common_params['checkpoint']) + + #### Logger + fuse_logger_start(output_path=paths['inference_dir'], console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Inference', {'attrs': ['bold', 'underline']}) + lgr.info(f'infer_filename={infer_file}', {'color': 'magenta'}) + lgr.info(f'infer folds={infer_common_params["data.infer_folds"]}', {'color': 'magenta'}) + + ## Data + folds = load_pickle(paths["data_split_filename"]) # assume exists and created in train func + + infer_sample_ids = [] + for fold in infer_common_params["data.infer_folds"]: + infer_sample_ids += folds[fold] + + params = dict(label_type=infer_common_params['label_type'], data_dir=paths["data_dir"], + cache_dir=paths["cache_dir"], train=False, + sample_ids=infer_sample_ids, verbose=False) + if not audit_cache: + params['cache_kwargs'] = dict(use_pipeline_hash=False, audit_first_sample=False, audit_rate=None) + else: + params['cache_kwargs'] = dict(use_pipeline_hash=False) + infer_dataset = prostate_x.ProstateX.dataset(**params) + + # dataloader + infer_dataloader = DataLoader(dataset=infer_dataset, batch_size=infer_common_params['data.batch_size'], collate_fn=CollateDefault(), + num_workers=infer_common_params['data.num_workers']) + + # load python lightning module + model = create_model(**infer_common_params["model"]) + pl_module = LightningModuleDefault.load_from_checkpoint(checkpoint_file, model_dir=paths["model_dir"], model=model, map_location="cpu", strict=True) + # set the prediction keys to extract (the ones used be the evaluation function). + pl_module.set_predictions_keys(['model.output.classification', 'data.ground_truth']) # which keys to extract and dump into file + + # create a trainer instance + pl_trainer = pl.Trainer(default_root_dir=paths['model_dir'], + accelerator=infer_common_params["trainer.accelerator"], + devices=infer_common_params["trainer.num_devices"], + auto_select_gpus=True) + predictions = pl_trainer.predict(pl_module, infer_dataloader, return_predictions=True) + + # convert list of batch outputs into a dataframe + infer_df = convert_predictions_to_dataframe(predictions) + save_dataframe(infer_df, infer_file) + + +###################################### +# Analyze Template +###################################### +def run_eval(paths: dict, eval_common_params: dict): + infer_file = os.path.join(paths['inference_dir'], eval_common_params['infer_filename']) + + fuse_logger_start(output_path=None, console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Analyze', {'attrs': ['bold', 'underline']}) + + # metrics + metrics = OrderedDict([ + ('operation_point', MetricApplyThresholds(pred='model.output.classification')), # will apply argmax + ('accuracy', MetricAccuracy(pred='results:metrics.operation_point.cls_pred', target='data.ground_truth')), + ('roc', MetricROCCurve(pred='model.output.classification', target='data.ground_truth', + output_filename=os.path.join(paths['inference_dir'], 'roc_curve.png'))), + ('auc', MetricAUCROC(pred='model.output.classification', target='data.ground_truth')), + ]) + + # create evaluator + evaluator = EvaluatorDefault() + + # run + results = evaluator.eval(ids=None, + data=infer_file, + metrics=metrics, + output_dir=paths['eval_dir']) + + return results + + +###################################### +# Run +###################################### +if __name__ == "__main__": + main() diff --git a/examples/fuse_examples/imaging/classification/prostate_x/runner_prostate_x_old_manager.py b/examples/fuse_examples/imaging/classification/prostate_x/runner_prostate_x_old_manager.py new file mode 100644 index 000000000..6da147a1b --- /dev/null +++ b/examples/fuse_examples/imaging/classification/prostate_x/runner_prostate_x_old_manager.py @@ -0,0 +1,482 @@ +""" +(C) Copyright 2021 IBM Corp. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Created on June 30, 2021 + +""" + +import logging +import os +from typing import OrderedDict, Optional + +import torch.nn.functional as F +import torch.optim as optim +from torch.utils.data.dataloader import DataLoader + +import fuse.utils.gpu as GPU +from fuse_fuse_examples_utils import ask_user +from fuse_examples.imaging.utils.backbone_3d_multichannel import Fuse_model_3d_multichannel, ResNet +from fuse.data.utils.collates import CollateDefault +from fuse.data.utils.samplers import BatchSamplerDefault +from fuse.data.utils.split import dataset_balanced_division_to_folds +from fuse.dl.losses.loss_default import LossDefault +from fuse.dl.managers.callbacks.callback_metric_statistics import MetricStatisticsCallback +from fuse.dl.managers.callbacks.callback_tensorboard import TensorboardCallback +from fuse.dl.managers.callbacks.callback_time_statistics import TimeStatisticsCallback +from fuse.dl.managers.manager_default import ManagerDefault +from fuse.dl.models.heads import Head1DClassifier +from fuse.eval.evaluator import EvaluatorDefault +from fuse.eval.metrics.classification.metrics_classification_common import MetricAccuracy, MetricAUCROC, MetricROCCurve +from fuse.eval.metrics.classification.metrics_thresholding_common import MetricApplyThresholds +from fuse.utils.file_io.file_io import load_pickle +from fuse.utils.rand.seed import Seed +from fuse.utils.utils_debug import FuseDebug +from fuse.utils.utils_logger import fuse_logger_start +from fuseimg.datasets import prostate_x + +from fuse.data.utils.export import ExportDataset +from fuse_examples import fuse_examples_utils + +def main(): + mode = 'default' # Options: 'default', 'fast', 'debug', 'verbose', 'user'. See details in FuseDebug + + # allocate gpus + # To use cpu - set NUM_GPUS to 0 + if mode == 'debug': + NUM_GPUS = 1 + else: + NUM_GPUS = 2 + # uncomment if you want to use specific gpus instead of automatically looking for free ones + force_gpus = None # [0] + GPU.choose_and_enable_multiple_gpus(NUM_GPUS, force_gpus=force_gpus) + + PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS = get_setting(mode, n_folds=8, + heldout_fold=4) + print(PATHS) + + RUNNING_MODES = ['train', 'infer', 'eval'] # Options: 'train', 'infer', 'eval' + # train + if 'train' in RUNNING_MODES: + print(TRAIN_COMMON_PARAMS) + run_train(paths=PATHS, train_params=TRAIN_COMMON_PARAMS) + + # infer + if 'infer' in RUNNING_MODES: + print(INFER_COMMON_PARAMS) + run_infer(paths=PATHS, infer_common_params=INFER_COMMON_PARAMS, audit_cache='train' not in RUNNING_MODES) + + # eval + if 'eval' in RUNNING_MODES: + print(EVAL_COMMON_PARAMS) + run_eval(paths=PATHS, eval_common_params=EVAL_COMMON_PARAMS) + + print(f"Done running with heldout={INFER_COMMON_PARAMS['data.infer_folds']}") + +def get_setting(mode, label_type=prostate_x.ProstateXLabelType.ClinSig, n_folds=8, heldout_fold=7, num_epoch=None): + ########################################################################################################### + # Fuse + ########################################################################################################### + ########################################## + # Debug modes + ########################################## + + debug = FuseDebug(mode) + input_channels_num = 5 + ########################################## + # Output Paths + ########################################## + assert "PROSTATEX_DATA_PATH" in os.environ, "Expecting environment variable PROSTATEX_DATA_PATH to be set. Follow the instruction in example README file to download and set the path to the data" + data_dir = os.environ["PROSTATEX_DATA_PATH"] + ROOT = os.path.join(fuse_examples_utils.get_fuse_examples_user_dir(), 'prostate_x') + + if mode == 'debug': + data_split_file = os.path.join(ROOT, f'prostate_x_{n_folds}folds_debug.pkl') + selected_sample_ids = prostate_x.get_samples_for_debug(n_pos=10, n_neg=10, label_type=label_type) + print(selected_sample_ids) + cache_dir = os.path.join(ROOT, 'cache_dir_debug') + model_dir = os.path.join(ROOT, 'model_dir_debug') + num_workers = 0 + batch_size = 2 + if num_epoch is None: + num_epoch = 5 + else: + data_split_file = os.path.join(ROOT, f'prostatex_{n_folds}folds.pkl') + cache_dir = os.path.join(ROOT, f'cache_dir') + model_dir = os.path.join(ROOT, f'model_dir_{heldout_fold}') + selected_sample_ids = None + + num_workers = 16 + batch_size = 50 + if num_epoch is None: + num_epoch = 50 + PATHS = {'model_dir': model_dir, + 'force_reset_model_dir': True, # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. + 'cache_dir': cache_dir, + 'data_split_filename': os.path.join(ROOT, data_split_file), + 'data_dir': data_dir, + 'inference_dir': os.path.join(model_dir, 'infer_dir'), + 'eval_dir': os.path.join(model_dir, 'eval_dir'), + } + + ########################################## + # Train Common Params + ########################################## + TRAIN_COMMON_PARAMS = {} + # ============ + # Model + # ============ + + # ============ + # Data + # ============ + + train_folds = [i % n_folds for i in range(heldout_fold + 1, heldout_fold + n_folds - 1)] + validation_fold = (heldout_fold - 1) % n_folds + TRAIN_COMMON_PARAMS['data.selected_sample_ids'] = selected_sample_ids + TRAIN_COMMON_PARAMS['data.batch_size'] = batch_size + TRAIN_COMMON_PARAMS['data.train_num_workers'] = num_workers + TRAIN_COMMON_PARAMS['data.validation_num_workers'] = num_workers + TRAIN_COMMON_PARAMS['data.num_folds'] = n_folds + TRAIN_COMMON_PARAMS['data.train_folds'] = train_folds + TRAIN_COMMON_PARAMS['data.validation_folds'] = [validation_fold] + + # =============== + # Manager - Train + # =============== + TRAIN_COMMON_PARAMS['manager.train_params'] = { + 'num_epochs': num_epoch, + 'virtual_batch_size': 1, # number of batches in one virtual batch + 'start_saving_epochs': 10, # first epoch to start saving checkpoints from + 'gap_between_saving_epochs': 5, # number of epochs between saved checkpoint + } + TRAIN_COMMON_PARAMS['manager.best_epoch_source'] = { + 'source': 'metrics.auc', # can be any key from 'epoch_results' + 'optimization': 'max', # can be either min/max + 'on_equal_values': 'better', + # can be either better/worse - whether to consider best epoch when values are equal + } + TRAIN_COMMON_PARAMS['manager.learning_rate'] = 1e-5 + TRAIN_COMMON_PARAMS['manager.weight_decay'] = 1e-4 + TRAIN_COMMON_PARAMS['manager.dropout'] = 0.5 + TRAIN_COMMON_PARAMS['manager.momentum'] = 0.9 + TRAIN_COMMON_PARAMS['manager.resume_checkpoint_filename'] = None # if not None, will try to load the checkpoint + TRAIN_COMMON_PARAMS['imaging_dropout'] = 0.25 + # TRAIN_COMMON_PARAMS['fused_dropout'] = 0.0 + # TRAIN_COMMON_PARAMS['clinical_dropout'] = 0.0 + + TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] = 512 + + # in order to add relevant tabular feature uncomment: + # num_backbone_features_clinical, post_concat_inputs,post_concat_model + TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] = None # 256 + TRAIN_COMMON_PARAMS['post_concat_inputs'] = None # [('data.clinical_features',9),] + TRAIN_COMMON_PARAMS['post_concat_model'] = None # (256,256) + + if TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] is None: + TRAIN_COMMON_PARAMS['num_backbone_features'] = TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] + else: + TRAIN_COMMON_PARAMS['num_backbone_features'] = \ + TRAIN_COMMON_PARAMS['num_backbone_features_imaging'] + TRAIN_COMMON_PARAMS['num_backbone_features_clinical'] + + # classification task: + # supported labels are: 'ClinSig' + TRAIN_COMMON_PARAMS['label_type'] = label_type + TRAIN_COMMON_PARAMS['class_num'] = label_type.get_num_classes() + + # backbone parameters + TRAIN_COMMON_PARAMS['backbone_model_dict'] = \ + {'input_channels_num': input_channels_num, + } + + ###################################### + # Inference Common Params + ###################################### + INFER_COMMON_PARAMS = {} + INFER_COMMON_PARAMS['infer_filename'] = 'validation_set_infer.gz' + INFER_COMMON_PARAMS['checkpoint'] = 'best' # Fuse TIP: possible values are 'best', 'last' or epoch_index. + INFER_COMMON_PARAMS['data.infer_folds'] = [heldout_fold] # infer validation set + INFER_COMMON_PARAMS['data.batch_size'] = 4 + INFER_COMMON_PARAMS['data.num_workers'] = num_workers + INFER_COMMON_PARAMS['label_type'] = TRAIN_COMMON_PARAMS['label_type'] + + ###################################### + # Analyze Common Params + ###################################### + EVAL_COMMON_PARAMS = {} + EVAL_COMMON_PARAMS['infer_filename'] = INFER_COMMON_PARAMS['infer_filename'] + + return PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS + + +################################# +# Train Template +################################# +def run_train(paths: dict, train_params: dict, reset_cache=None, audit_cache=None): + Seed.set_seed(222, False) + + # ============================================================================== + # Logger + # ============================================================================== + fuse_logger_start(output_path=paths['model_dir'], console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Train', {'attrs': ['bold', 'underline']}) + + lgr.info(f'model_dir={paths["model_dir"]}', {'color': 'magenta'}) + lgr.info(f'cache_dir={paths["cache_dir"]}', {'color': 'magenta'}) + lgr.info(f'train folds={train_params["data.train_folds"]}', {'color': 'magenta'}) + lgr.info(f'validation folds={train_params["data.validation_folds"]}', {'color': 'magenta'}) + + + # ============================================================================== + # Data + # ============================================================================== + # Train Data + lgr.info(f'Train Data:', {'attrs': 'bold'}) + + if reset_cache is None: + reset_cache = ask_user('Do you want to reset cache?') + cache_kwargs = {'use_pipeline_hash': False} + if not reset_cache: + if audit_cache is None: + audit_cache = ask_user('Do you want to audit cache?') + if not audit_cache: + cache_kwargs2 = dict(audit_first_sample=False, audit_rate=None) + cache_kwargs = {**cache_kwargs, **cache_kwargs2} + + # split to folds randomly + params = dict(label_type=train_params['label_type'], data_dir=paths["data_dir"], cache_dir=paths["cache_dir"], + reset_cache=reset_cache, sample_ids=train_params['data.selected_sample_ids'], + num_workers=train_params['data.train_num_workers'], + cache_kwargs=cache_kwargs, train=False, verbose=False) + + dataset_all = prostate_x.ProstateX.dataset(**params) + # ExportDataset.export_to_dir(dataset=dataset_all, output_dir=f'/tmp/ozery/prostatex_{my_version}') + + folds = dataset_balanced_division_to_folds(dataset=dataset_all, + output_split_filename=paths["data_split_filename"], + keys_to_balance=["data.ground_truth"], + id='data.input.patient_id', + workers=0, #todo: stuck in Export to dataframe + nfolds=train_params["data.num_folds"], + verbose=True) + + train_sample_ids = [] + for fold in train_params["data.train_folds"]: + train_sample_ids += folds[fold] + validation_sample_ids = [] + for fold in train_params["data.validation_folds"]: + validation_sample_ids += folds[fold] + + params['sample_ids'] = train_sample_ids + params['reset_cache'] = False + params['train'] = True + params['cache_kwargs'] = dict(use_pipeline_hash=False, audit_first_sample=False, audit_rate=None) + train_dataset = prostate_x.ProstateX.dataset(**params) + # for _ in train_dataset: + # pass + params['sample_ids'] = validation_sample_ids + params['train'] = False + validation_dataset = prostate_x.ProstateX.dataset(**params) + + lgr.info(f'- Create sampler:') + sampler = BatchSamplerDefault(dataset=train_dataset, + balanced_class_name='data.ground_truth', + num_balanced_classes=train_params['class_num'], + batch_size=train_params['data.batch_size'], + workers=0 #train_params['data.train_num_workers'] #todo: stuck + ) + lgr.info(f'- Create sampler: Done') + + # Create dataloader + train_dataloader = DataLoader(dataset=train_dataset, + batch_sampler=sampler, + collate_fn=CollateDefault(), + num_workers=train_params['data.train_num_workers']) + lgr.info(f'Train Data: Done', {'attrs': 'bold'}) + + # dataloader + validation_dataloader = DataLoader(dataset=validation_dataset, + batch_size=train_params['data.batch_size'], + collate_fn=CollateDefault(), + num_workers=train_params['data.validation_num_workers']) + lgr.info(f'Validation Data: Done', {'attrs': 'bold'}) + + # ============================================================================== + # Model + # ============================================================================== + lgr.info('Model:', {'attrs': 'bold'}) + + conv_inputs = (('data.input.patch_volume', 1),) + model = Fuse_model_3d_multichannel( + conv_inputs=conv_inputs, # previously 'data.input'. could be either 'data.input.patch_volume' or 'data.input.patch_volume_orig' + backbone=ResNet(conv_inputs=conv_inputs, ch_num=train_params['backbone_model_dict']['input_channels_num']), + # since backbone resnet contains pooling and fc, the feature output is 1D, + # hence we use Head1dClassifier as classification head + heads=[ + Head1DClassifier(head_name='classification', + conv_inputs=[('model.backbone_features', train_params['num_backbone_features'])], + post_concat_inputs=train_params['post_concat_inputs'], + post_concat_model=train_params['post_concat_model'], + dropout_rate=train_params['imaging_dropout'], + # append_dropout_rate=train_params['clinical_dropout'], + # fused_dropout_rate=train_params['fused_dropout'], + shared_classifier_head=None, + layers_description=None, + num_classes=2, + # append_features=[("data.input.clinical", 8)], + # append_layers_description=(256,128), + ), + ] + ) + + lgr.info('Model: Done', {'attrs': 'bold'}) + + # ==================================================================================== + # Loss + # ==================================================================================== + losses = { + 'cls_loss': LossDefault(pred='model.logits.classification', + target='data.ground_truth', callable=F.cross_entropy, weight=1.0), + } + + # ==================================================================================== + # Metrics + # ==================================================================================== + lgr.info('Metrics:', {'attrs': 'bold'}) + metrics = OrderedDict([ + ('auc', MetricAUCROC(pred='model.output.classification', target='data.ground_truth')) + ]) + + # ===================================================================================== + # Callbacks + # ===================================================================================== + callbacks = [ + # default callbacks + TensorboardCallback(model_dir=paths['model_dir']), # save statistics for tensorboard + MetricStatisticsCallback(output_path=paths['model_dir'] + "/metrics.csv"), # save statistics a csv file + TimeStatisticsCallback(num_epochs=train_params['manager.train_params']['num_epochs'], load_expected_part=0.1) # time profiler + ] + + # ===================================================================================== + # Manager - Train + # ===================================================================================== + lgr.info('Train:', {'attrs': 'bold'}) + + # create optimizer + optimizer = optim.Adam(model.parameters(), lr=train_params['manager.learning_rate'], + weight_decay=train_params['manager.weight_decay']) + + # create learning scheduler + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True) + + # train from scratch + manager = ManagerDefault(output_model_dir=paths['model_dir'], force_reset=paths['force_reset_model_dir']) + # Providing the objects required for the training process. + manager.set_objects(net=model, + optimizer=optimizer, + losses=losses, + metrics=metrics, + best_epoch_source=train_params['manager.best_epoch_source'], + lr_scheduler=scheduler, + callbacks=callbacks, + train_params=train_params['manager.train_params']) + + ## Continue training + if train_params['manager.resume_checkpoint_filename'] is not None: + # Loading the checkpoint including model weights, learning rate, and epoch_index. + manager.load_checkpoint(checkpoint=train_params['manager.resume_checkpoint_filename'], mode='train') + + # Start training + manager.train(train_dataloader=train_dataloader, validation_dataloader=validation_dataloader) + + lgr.info('Train: Done', {'attrs': 'bold'}) + + +###################################### +# Inference Template +###################################### +def run_infer(paths: dict, infer_common_params: dict, audit_cache: Optional[bool] = True): + #### Logger + fuse_logger_start(output_path=paths['inference_dir'], console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Inference', {'attrs': ['bold', 'underline']}) + lgr.info(f'infer_filename={os.path.join(paths["inference_dir"], infer_common_params["infer_filename"])}', {'color': 'magenta'}) + lgr.info(f'infer folds={infer_common_params["data.infer_folds"]}', {'color': 'magenta'}) + + ## Data + folds = load_pickle(paths["data_split_filename"]) # assume exists and created in train func + + infer_sample_ids = [] + for fold in infer_common_params["data.infer_folds"]: + infer_sample_ids += folds[fold] + + + params = dict(label_type=infer_common_params['label_type'], data_dir=paths["data_dir"], + cache_dir=paths["cache_dir"], train=False, + sample_ids=infer_sample_ids, verbose=False) + if not audit_cache: + params['cache_kwargs'] =dict(use_pipeline_hash=False, audit_first_sample=False, audit_rate=None) + else: + params['cache_kwargs'] = dict(use_pipeline_hash=False) + validation_dataset = prostate_x.ProstateX.dataset(**params) + + # dataloader + validation_dataloader = DataLoader(dataset=validation_dataset, batch_size=infer_common_params['data.batch_size'], collate_fn=CollateDefault(), + num_workers=infer_common_params['data.num_workers']) + + ## Manager for inference + manager = ManagerDefault() + output_columns = ['model.output.classification', 'data.ground_truth'] + manager.infer(data_loader=validation_dataloader, + input_model_dir=paths['model_dir'], + checkpoint=infer_common_params['checkpoint'], + output_columns=output_columns, + output_file_name=os.path.join(paths["inference_dir"], infer_common_params["infer_filename"])) + + +###################################### +# Analyze Template +###################################### +def run_eval(paths: dict, eval_common_params: dict): + fuse_logger_start(output_path=None, console_verbose_level=logging.INFO) + lgr = logging.getLogger('Fuse') + lgr.info('Fuse Analyze', {'attrs': ['bold', 'underline']}) + + # metrics + metrics = OrderedDict([ + ('operation_point', MetricApplyThresholds(pred='model.output.classification')), # will apply argmax + ('accuracy', MetricAccuracy(pred='results:metrics.operation_point.cls_pred', target='data.ground_truth')), + ('roc', MetricROCCurve(pred='model.output.classification', target='data.ground_truth', + output_filename=os.path.join(paths['inference_dir'], 'roc_curve.png'))), + ('auc', MetricAUCROC(pred='model.output.classification', target='data.ground_truth')), + ]) + + # create evaluator + evaluator = EvaluatorDefault() + + # run + results = evaluator.eval(ids=None, + data=os.path.join(paths["inference_dir"], eval_common_params["infer_filename"]), + metrics=metrics, + output_dir=paths['eval_dir']) + + return results + + +###################################### +# Run +###################################### +if __name__ == "__main__": + main() diff --git a/examples/fuse_examples/imaging/classification/prostate_x/tasks.py b/examples/fuse_examples/imaging/classification/prostate_x/tasks.py deleted file mode 100644 index 9a22308fa..000000000 --- a/examples/fuse_examples/imaging/classification/prostate_x/tasks.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -(C) Copyright 2021 IBM Corp. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -Created on June 30, 2021 -""" - -import logging -from typing import List - - -class ProstateXTask(): - tasks = {} - def __init__(self, task_name: str, version: int): - self._task_name, self._task_version, self._task_mapping, self._task_class_names = \ - self.get_task(task_name, version) - - def name(self): - return self._task_name + "_" + str(self._task_version) - - def class_names(self): - return self._task_class_names - - def num_classes(self): - return len(self._task_class_names) - - def mapping(self): - return self._task_mapping - - @classmethod - def register(cls, name: str, version: int, mapping: List, class_names: List[str]): - key = (name, version) - assert key not in cls.tasks - cls.tasks[key] = (name, version, mapping, class_names) - - @classmethod - def get_task(cls, task_name: str, version: int): - key = (task_name, version) - if key not in cls.tasks: - msg = f'Task not found - list of tasks: {list(cls.tasks.keys())}' - logging.getLogger('Fuse').error(msg) - raise Exception(msg) - - return cls.tasks[key] - - - -#DO NOT CHANGE TASKS!!!! -GLEASON_SCORE = ['HIGH','LOW','BENIGN'] -GLEASON_SCORE_VER_0 = [['HIGH'], ['LOW'],['BENIGN']], -CLINSIG_VER_0 = [['HIGH'], ['LOW']], - - -ProstateXTask.register('gleason_score', 0, GLEASON_SCORE_VER_0, ['HIGH','LOW','BENIGN']) -ProstateXTask.register('ClinSig', 0, CLINSIG_VER_0, ['HIGH','LOW']) -if __name__ == '__main__': - mp_task = ProstateXTask('gleason_score', 0) - print(mp_task.name()) - print(mp_task.class_names()) - print(len(mp_task.class_names())) - print(mp_task.mapping()) \ No newline at end of file diff --git a/examples/fuse_examples/imaging/utils/__init__.py b/examples/fuse_examples/imaging/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/fuse_examples/imaging/classification/prostate_x/backbone_3d_multichannel.py b/examples/fuse_examples/imaging/utils/backbone_3d_multichannel.py similarity index 94% rename from examples/fuse_examples/imaging/classification/prostate_x/backbone_3d_multichannel.py rename to examples/fuse_examples/imaging/utils/backbone_3d_multichannel.py index 6195f4b4c..f295e2d18 100644 --- a/examples/fuse_examples/imaging/classification/prostate_x/backbone_3d_multichannel.py +++ b/examples/fuse_examples/imaging/utils/backbone_3d_multichannel.py @@ -166,19 +166,13 @@ def forward(self, batch_dict: NDict): class Fuse_model_3d_multichannel(torch.nn.Module): """ Fuse model that classifing high resolution images - Sequence: - 1. Starts with low resolution image to extract low resolution features and attention score per each image patch - 2. Extract high resolution features 'k' most significant patches (using the attention score) - 3. Use low resolution features, high resolution features and attention score to classify the image - Input: high resolution tensor [BATCH_SIZE, 1, H, W] - Output: - TBD + """ def __init__(self, conv_inputs: Tuple[Tuple[str, int], ...] = (('data.input', 1),), - backbone: ResNet = ResNet(), - heads: Sequence[torch.nn.Module] = (Head1DClassifier(),), + backbone: ResNet = None, + heads: Sequence[torch.nn.Module] = None, ch_num = None, ) -> None: """ @@ -190,6 +184,10 @@ def __init__(self, super().__init__() self.conv_inputs = conv_inputs + if backbone is None: + backbone = ResNet(conv_inputs=conv_inputs) + if heads is None: + heads = (Head1dClassifier(),) self.backbone = backbone self.heads = torch.nn.ModuleList(heads) self.add_module('heads', self.heads) diff --git a/examples/fuse_examples/tests/test_classification_duke.py b/examples/fuse_examples/tests/test_classification_duke.py new file mode 100644 index 000000000..97928a85a --- /dev/null +++ b/examples/fuse_examples/tests/test_classification_duke.py @@ -0,0 +1,81 @@ +""" +(C) Copyright 2021 IBM Corp. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Created on June 30, 2021 + +""" + +import shutil +import tempfile +import unittest +import os +from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess + +from fuse.utils.rand.seed import Seed +import fuse.utils.gpu as GPU + +# if "DUKE_DATA_PATH" in os.environ: +# from fuse_examples.imaging.classification.duke_breast_cancer.runner_duke import get_setting, run_train, run_infer, run_eval + +# @unittest.skipIf("DUKE_DATA_PATH" not in os.environ, "define environment variable 'DUKE_DATA_PATH' to run this test") +@unittest.skipIf(True) +class ClassificationDukeTestCase(unittest.TestCase): + + def setUp(self): + selected_positive = [1,2,3,5,6,10,12,596, 900, 901] + selected_negative = [4,6,7,8,11,13,14,120, 902, 903] + + selected_sample_ids = [f'Breast_MRI_{ii:03d}' for ii in selected_positive + selected_negative] + PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS = get_setting('default', + selected_sample_ids=selected_sample_ids, + num_epoch=2) + + self.root = tempfile.mkdtemp() + + self.paths = { + 'model_dir': os.path.join(self.root, 'duke/model_dir'), + 'force_reset_model_dir': True, # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. + 'data_dir': PATHS["data_dir"], + 'cache_dir': os.path.join(self.root, 'duke/cache_dir'), + 'data_split_filename': os.path.join(self.root, 'split.pkl'), + 'inference_dir': os.path.join(self.root, 'duke/infer_dir'), + 'eval_dir': os.path.join(self.root, 'duke/analyze_dir')} + + + self.train_common_params = TRAIN_COMMON_PARAMS + # self.train_common_params["manager.train_params"]["num_epochs"] = 2 + self.infer_common_params = INFER_COMMON_PARAMS + + self.analyze_common_params = EVAL_COMMON_PARAMS + + + @run_in_subprocess(1200) + def test_template(self): + GPU.choose_and_enable_multiple_gpus(1) + + Seed.set_seed(0, False) # previous test (in the pipeline) changed the deterministic behavior to True + run_train(self.paths, self.train_common_params, reset_cache=True, audit_cache=False) + run_infer(self.paths, self.infer_common_params, audit_cache=False) + results = run_eval(self.paths, self.analyze_common_params) + + self.assertTrue('metrics.auc' in results) + + def tearDown(self): + # Delete temporary directories + shutil.rmtree(self.root) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/examples/fuse_examples/tests/test_classification_knight.py b/examples/fuse_examples/tests/test_classification_knight.py index e008d719e..f7f898285 100644 --- a/examples/fuse_examples/tests/test_classification_knight.py +++ b/examples/fuse_examples/tests/test_classification_knight.py @@ -27,9 +27,9 @@ # FIXME: data_package -#from fuse_examples.imaging.classification.knight.eval.eval import eval -#from fuse_examples.imaging.classification.knight.make_targets_file import make_targets_file -#import fuse_examples.imaging.classification.knight.baseline.fuse_baseline as baseline +from fuse_examples.imaging.classification.knight.eval.eval import eval +from fuse_examples.imaging.classification.knight.make_targets_file import make_targets_file +import fuse_examples.imaging.classification.knight.baseline.fuse_baseline as baseline @unittest.skip("FIXME: data_package") class KnightTestTestCase(unittest.TestCase): diff --git a/examples/fuse_examples/tests/test_classification_prostatex.py b/examples/fuse_examples/tests/test_classification_prostatex.py index c39acc8d8..d6fa94f8d 100644 --- a/examples/fuse_examples/tests/test_classification_prostatex.py +++ b/examples/fuse_examples/tests/test_classification_prostatex.py @@ -21,31 +21,29 @@ import tempfile import unittest import os -import pathlib +from fuse.utils.multiprocessing.run_multiprocessed import run_in_subprocess +from fuse.utils.rand.seed import Seed import fuse.utils.gpu as GPU -# FIXME: data_package -#from fuse_examples.imaging.classification.prostate_x.run_train_3dpatch import TRAIN_COMMON_PARAMS, train_template, infer_template, eval_template, INFER_COMMON_PARAMS, \ -# EVAL_COMMON_PARAMS +from fuse_examples.imaging.classification.prostate_x.runner_prostate_x_old_manager import get_setting, run_train, run_infer, run_eval +# @unittest.skipIf("PROSTATEX_DATA_PATH" not in os.environ, "define environment variable 'PROSTATEX_DATA_PATH' to run this test") +@unittest.skipIf(True) class ClassificationProstateXTestCase(unittest.TestCase): def setUp(self): + PATHS, TRAIN_COMMON_PARAMS, INFER_COMMON_PARAMS, EVAL_COMMON_PARAMS = get_setting('default',num_epoch=5) self.root = tempfile.mkdtemp() - root_path = self.root - root_data = '/projects/msieve/MedicalSieve/PatientData/ProstateX/manifest-A3Y4AE4o5818678569166032044/' - self.paths = {'force_reset_model_dir': True, - # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. - 'model_dir': os.path.join(root_path, 'prostatex/my_model/'), - 'cache_dir': os.path.join(root_path, 'prostatex/my_cache/'), - 'inference_dir': os.path.join(root_path, 'prostatex/my_model/inference/'), - 'eval_dir': os.path.join(root_path, 'prostatex/my_model/eval/'), - 'data_dir': os.path.join(pathlib.Path(__file__).parent.resolve(), "../classification/prostate_x"), - 'prostate_data_path' : root_data, - 'ktrans_path': os.path.join(root_data, 'ProstateXKtrains-train-fixed/'), - } + self.paths = { + 'model_dir': os.path.join(self.root, 'prostatex/model_dir'), + 'force_reset_model_dir': True, # If True will reset model dir automatically - otherwise will prompt 'are you sure' message. + 'data_dir': PATHS["data_dir"], + 'cache_dir': os.path.join(self.root, 'prostatex/cache_dir'), + 'data_split_filename': os.path.join(self.root, 'split.pkl'), + 'inference_dir': os.path.join(self.root, 'prostatex/infer_dir'), + 'eval_dir': os.path.join(self.root, 'prostatex/analyze_dir')} self.train_common_params = TRAIN_COMMON_PARAMS @@ -54,20 +52,17 @@ def setUp(self): self.analyze_common_params = EVAL_COMMON_PARAMS - @unittest.skip("Not ready yet") - # TODO: - # 1. Get path as an env variable - # 2. modify the result value check + + @run_in_subprocess(1200) def test_template(self): - num_gpus_allocated = GPU.choose_and_enable_multiple_gpus(1, use_cpu_if_fail=True) - if num_gpus_allocated == 0: - self.train_common_params['manager.train_params']['device'] = 'cpu' - train_template(self.paths, self.train_common_params) - infer_template(self.paths, self.infer_common_params) - results = eval_template(self.paths, self.analyze_common_params) - - threshold = 0.98 - self.assertGreaterEqual(results['metrics.auc.macro_avg'], threshold) + GPU.choose_and_enable_multiple_gpus(1) + + Seed.set_seed(0, False) # previous test (in the pipeline) changed the deterministic behavior to True + run_train(self.paths, self.train_common_params, reset_cache=True, audit_cache=False) + run_infer(self.paths, self.infer_common_params, audit_cache=False) + results = run_eval(self.paths, self.analyze_common_params) + + self.assertTrue('metrics.auc' in results) def tearDown(self): # Delete temporary directories diff --git a/fuse/data/datasets/caching/samples_cacher.py b/fuse/data/datasets/caching/samples_cacher.py index 953d6b46f..ac0207bab 100644 --- a/fuse/data/datasets/caching/samples_cacher.py +++ b/fuse/data/datasets/caching/samples_cacher.py @@ -43,6 +43,7 @@ def __init__(self, restart_cache:bool=False, workers:int = 0, verbose=1, + use_pipeline_hash: Optional[bool] = True, **audit_kwargs:dict, ) -> None: """ @@ -60,6 +61,7 @@ def __init__(self, Should be used every time that any of the OPs participating in the "static cache" part changed in any way (for example, code change) :param workers: number of multiprocessing workers used when building the cache. Default value is 0 (no multiprocessing) + :param use_pipeline_hash [Optional]: indicates whether to use a hash of given pipeline for naming its cache dir. Default=True :param **audit_kwargs: optional custom kwargs to pass to SampleCachingAudit instance. auditing cached samples (usually periodically) is very important, in order to avoid "stale" cached samples. To disable pass audit_first_sample=False, audit_rate=None, @@ -84,8 +86,12 @@ def __init__(self, self._read_dirs_logic = custom_read_dirs_callable self._pipeline = pipeline + self._use_pipeline_hash = use_pipeline_hash self._pipeline_desc_text = str(pipeline) - self._pipeline_desc_hash = 'hash_'+hashlib.md5(self._pipeline_desc_text.encode('utf-8')).hexdigest() + if use_pipeline_hash: + self._pipeline_desc_hash = 'hash_'+hashlib.md5(self._pipeline_desc_text.encode('utf-8')).hexdigest() + else: + self._pipeline_desc_hash = 'hash_fixed' self._verbose = verbose @@ -97,6 +103,8 @@ def __init__(self, self.delete_cache() self._audit_kwargs = audit_kwargs + if 'ignore_nan_inequality' not in self._audit_kwargs: + self._audit_kwargs['ignore_nan_inequality'] = True #as DeepDiff(float('nan'), float('nan')) reports difference self._audit = SampleCachingAudit(**self._audit_kwargs) self._workers = workers diff --git a/fuse/data/ops/ops_common.py b/fuse/data/ops/ops_common.py index f43fc0adc..032bf058d 100644 --- a/fuse/data/ops/ops_common.py +++ b/fuse/data/ops/ops_common.py @@ -89,18 +89,25 @@ def __init__(self, self._func = func self._func_reverse = func_reverse - def __call__(self, sample_dict: NDict, op_id: Optional[str], key: Optional[str] = None, **kwargs) -> Union[None, dict, List[dict]]: + def __call__(self, sample_dict: NDict, op_id: Optional[str], key: Optional[str] = None, key_out:Optional[str]=None, **kwargs) -> Union[None, dict, List[dict]]: """ More details in super class - :param key: apply lambda func on sample_dict[key]. If none the input and output of the lambda function are the entire sample_dict + :param key [Optional]: apply lambda func on sample_dict[key]. If none (default) the input and output of the lambda function are the entire sample_dict + :param key_out [Optional]: sample_dict[key_out] = func(sample_dict[key]) """ sample_dict[op_id] = key if key is not None: value = sample_dict[key] value = self._func(value, **kwargs) - sample_dict[key] = value + res = value else: - sample_dict = self._func(sample_dict) + res = self._func(sample_dict) + if key_out is not None: + sample_dict[key_out] = res + elif key is not None: + sample_dict[key] = res + else: + sample_dict = res return sample_dict @@ -351,9 +358,8 @@ def __init__(self, **kwargs): def __call__(self, sample_dict: NDict, keep_keypaths:List[str]) -> Union[None, dict, List[dict]]: prev_sample_dict = sample_dict sample_dict = NDict() - for k in keep_keypaths: + for k in ['data.initial_sample_id', 'data.sample_id']+keep_keypaths: sample_dict[k] = prev_sample_dict[k] - return sample_dict @@ -382,7 +388,7 @@ def __call__(self, sample_dict: NDict, key_in: str, key_out: str) -> Union[None, sample_dict[key_out] = self._map[value] elif self._not_exist_error: raise Exception(f"value {value} does not exist in mapping") - + return sample_dict diff --git a/fuse/data/ops/ops_read.py b/fuse/data/ops/ops_read.py index 4dc358238..d321b0232 100644 --- a/fuse/data/ops/ops_read.py +++ b/fuse/data/ops/ops_read.py @@ -80,7 +80,7 @@ def __init__(self, df = df.set_index(self._key_column) self._data = df.to_dict(orient='index') - def __call__(self, sample_dict: NDict, **kwargs) -> Union[None, dict, List[dict]]: + def __call__(self, sample_dict: NDict, key_out_group=None, **kwargs) -> Union[None, dict, List[dict]]: """ See base class """ @@ -90,8 +90,11 @@ def __call__(self, sample_dict: NDict, **kwargs) -> Union[None, dict, List[dict] sample_data = self._data[key].copy() # add values tp sample_dict - for name, value in sample_data.items(): - sample_dict[name] = value + if key_out_group is None: + for name, value in sample_data.items(): + sample_dict[name] = value + else: + sample_dict[key_out_group] = sample_data return sample_dict diff --git a/fuse/data/pipelines/pipeline_default.py b/fuse/data/pipelines/pipeline_default.py index a3bf4c843..282b264c2 100644 --- a/fuse/data/pipelines/pipeline_default.py +++ b/fuse/data/pipelines/pipeline_default.py @@ -87,7 +87,7 @@ def __call__(self, sample_dict: NDict, op_id: Optional[str] = None, until_op_id: samples_to_process = [sample_dict] for sub_op_id, (op, op_kwargs) in zip(self._op_ids, self._ops_and_kwargs): if self._verbose: - context = Timer(f"Pipeline {self._name}: op {type(op).__name__}, op_id {sub_op_id}", self._verbose) + context = Timer(f"Pipeline {self._name}: op {type(op).__name__}, op_id {sub_op_id}") else: context = DummyContext() with context: diff --git a/fuse/data/utils/export.py b/fuse/data/utils/export.py index f584327be..c5aa407f9 100644 --- a/fuse/data/utils/export.py +++ b/fuse/data/utils/export.py @@ -18,10 +18,11 @@ """ from typing import Optional, Sequence import pandas as pds - +import torch +import os from fuse.data.datasets.dataset_base import DatasetBase -from fuse.utils.file_io.file_io import save_dataframe +from fuse.utils.file_io.file_io import save_dataframe, save_pickle_safe, load_pickle class ExportDataset: """ @@ -29,7 +30,9 @@ class ExportDataset: """ @staticmethod - def export_to_dataframe(dataset: DatasetBase, keys: Sequence[str], output_filename: Optional[str] = None, sample_id_key: str = "data.sample_id", **dataset_get_kwargs) -> pds.DataFrame: + def export_to_dataframe(dataset: DatasetBase, keys: Sequence[str], output_filename: Optional[str] = None, + sample_id_key: str = "data.sample_id", + **dataset_get_kwargs) -> pds.DataFrame: """ extract from dataset the specified and keys and create a dataframe. If output_filename will be specified, the dataframe will also be saved in a file. @@ -63,3 +66,47 @@ def export_to_dataframe(dataset: DatasetBase, keys: Sequence[str], output_filena save_dataframe(df, output_filename) return df + + + @staticmethod + def export_to_dir(dataset: DatasetBase, output_dir: str, keys: Optional[Sequence[str]]=None, + sample_id_key: str = "data.sample_id", + **dataset_get_kwargs) : + """ + extract from dataset the specified and keys and writes to a specified directory in the disk + :param dataset: the dataset to extract the values from + :param keys: Optional, keys to extract from sample_dict. If None - all keys will be saved + :param output_dir: Optional, if set, will save the dataframe into a file. + :param dataset_get_kwargs: additional parameters to dataset.get(), might be used to optimize the running time + """ + # add sample_id to keys list + if keys is not None: + all_keys = [] + all_keys += list(keys) + if sample_id_key not in keys: + all_keys.append(sample_id_key) + else: + all_keys = None + + # read all the data + data = dataset.get_multi(keys=all_keys, **dataset_get_kwargs) + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + for i, sample_dict in enumerate(data): + sample_id = sample_dict[sample_id_key] + sample_dict = sample_dict.flatten() + + if all_keys is not None: + sample_dict = {sample_dict[k] for k in all_keys} + + + d2 = {} + for k, v in sample_dict.items(): + if isinstance(v, torch.Tensor): + v = v.numpy() + d2[k] = v + output_file = os.path.join(output_dir, f'{sample_id}.pkl') + save_pickle_safe(d2, output_file) + print(i, "wrote", output_file) + print("done") diff --git a/fuse/data/utils/samplers.py b/fuse/data/utils/samplers.py index b63dc9081..104df76e8 100644 --- a/fuse/data/utils/samplers.py +++ b/fuse/data/utils/samplers.py @@ -53,7 +53,7 @@ def __init__(self, :param balanced_class_weights: Optional, integer/float per balanced class, Expected length is num_balanced_classes. In mode 'exact' expecting list of integers that sums up to batch dict. In mode 'approx' expecting list of floats that sums up to ~1 - If not specified and equal number of samples from each class will be used. + If not specified - an equal number of samples from each class will be used. :param num_batches: optional - if set will force num_batches, otherwise num_batches will be automatically to go over each sample at least once (exactly or approximately). :param dataset_get_multi_kwargs: extra parameters for dataset.get_multi() to optimize the running time. """ diff --git a/fuse/data/utils/split.py b/fuse/data/utils/split.py index 7f0bf2976..89370a47e 100644 --- a/fuse/data/utils/split.py +++ b/fuse/data/utils/split.py @@ -13,8 +13,9 @@ """ import pickle +import logging import os -from typing import Hashable, Sequence +from typing import Hashable, Sequence, Optional from fuse.data.datasets.dataset_base import DatasetBase from fuse.utils.file_io.file_io import load_pickle, save_pickle from sklearn.utils import shuffle @@ -113,7 +114,9 @@ def balanced_division(df : pd.DataFrame, no_mixture_id : str, keys_to_balance: S -def dataset_balanced_division_to_folds(dataset: DatasetBase, output_split_filename: str, keys_to_balance: Sequence[str], nfolds: int, id:str=get_sample_id_key(), reset_split: bool = False, workers: int=10, mp_context : str =None, **kwargs): +def dataset_balanced_division_to_folds(dataset: DatasetBase, output_split_filename: str, keys_to_balance: Sequence[str], + nfolds: int, workers: int =10, mp_context : str =None, id:str =get_sample_id_key(), + reset_split: bool = False, verbose: Optional[bool]=False, **kwargs): """ Split dataset to folds. @@ -123,16 +126,25 @@ def dataset_balanced_division_to_folds(dataset: DatasetBase, output_split_filena :param keys_to_balance: balancing any possible combination of values. For example for ["data.gender", "data.cancer"], the algorithm will balance each one of the following groups between the folds. (gender=male, cancer=True), (gender=male, cancer=False), (gender=female, cancer=True), (gender=female, cancer=False) - :param nfolds : number of folds - :param id : id to balance the split by ( not allowed 2 in same fold) + :param nfolds: number of folds + :param id: id to balance the split by ( not allowed 2 in same fold) :param reset_split: delete output_split_filename and recompute the split - :param workers : numbers of workers for multiprocessing (eport dataset into dataframe) - :param mp_context : multiprocessing context: "fork", "spawn", etc. + :param verbose: Optional. Default=False + :param workers: numbers of workers for multiprocessing (eport dataset into dataframe) + :param mp_context: multiprocessing context: "fork", "spawn", etc. :param kwargs: more arguments controlling the split. See function balanced_division() for details - """ + """ + + if verbose: + lgr = logging.getLogger('Fuse') + if os.path.exists(output_split_filename) and not reset_split: + if verbose: + lgr.info(f"{output_split_filename} exists. Loading") return load_pickle(output_split_filename) else: + if verbose: + lgr.info(f"{output_split_filename} does not exists. Generating") if id == get_sample_id_key(): keys = [get_sample_id_key()] else: @@ -146,5 +158,7 @@ def dataset_balanced_division_to_folds(dataset: DatasetBase, output_split_filena for fold in range(nfolds): folds[fold] = list(df_folds[df_folds["fold"] == fold][get_sample_id_key()]) save_pickle(folds, output_split_filename) + if verbose: + lgr.info(f"wrote {output_split_filename}") return folds diff --git a/fuse/dl/models/backbones/backbone_inception_resnet_v2.py b/fuse/dl/models/backbones/backbone_inception_resnet_v2.py index c6c4ec6bf..b581a532a 100644 --- a/fuse/dl/models/backbones/backbone_inception_resnet_v2.py +++ b/fuse/dl/models/backbones/backbone_inception_resnet_v2.py @@ -23,6 +23,7 @@ import torch from torch.hub import load_state_dict_from_url import torch.nn as nn +from urllib.error import URLError def make_seq(foo: Callable, num: int, *args, **kwargs): @@ -309,6 +310,11 @@ def __init__(self, except AttributeError: logger = logging.getLogger('Fuse') logger.info('Invalid URL for InceptionResnetV2 pretrained weights') + except URLError as e: + logging.getLogger("Fuse").warning( + f"Couldn't load pretrained weights from the url: '{pretrained_weights_url}' due to the following URLError: '{e}'." + ) + # recreate the first conv with the required number of input parameters if input_channels_num != 3: diff --git a/fuse/utils/multiprocessing/run_multiprocessed.py b/fuse/utils/multiprocessing/run_multiprocessed.py index afa4fd509..e38c36c81 100644 --- a/fuse/utils/multiprocessing/run_multiprocessed.py +++ b/fuse/utils/multiprocessing/run_multiprocessed.py @@ -6,6 +6,7 @@ import multiprocessing as mp from termcolor import cprint import os +import traceback from collections.abc import Iterable import inspect @@ -248,6 +249,27 @@ def get_from_global_storage(key: str) -> Any: return _multiprocess_global_storage[key] +class Process(mp.Process): + def __init__(self, *args, **kwargs): + mp.Process.__init__(self, *args, **kwargs) + self._pconn, self._cconn = mp.Pipe() + self._exception = None + + def run(self): + try: + mp.Process.run(self) + self._cconn.send(None) + except Exception as e: + tb = traceback.format_exc() + self._cconn.send((e, tb)) + # raise e # You can still rise this exception if you need to + + @property + def exception(self): + if self._pconn.poll(): + self._exception = self._pconn.recv() + return self._exception + def run_in_subprocess(timeout: int = 600): """A decorator that makes function run in a subprocess. This can be useful when you want allocate GPU and memory and to release it when you're done. @@ -261,14 +283,30 @@ def wrapper(*args, **kwargs): # create the machinery python uses to fork a subprocess # and run a function in it. p = mp.Process(target=f, args=args, kwargs=kwargs) + # p = Process(target=f, args=args, kwargs=kwargs) # using a subclass of Process p.start() try: p.join(timeout=timeout) except: p.terminate() raise - + + #if p.exception: + # error, traceback = p.exception + # print(f"process func {f} had an exception: {error}") + # print(traceback) + # raise RuntimeError(f"process func {f} had an exception: {error}") + assert p.exitcode == 0, f"process func {f} failed with exit code {p.exitcode}" return wrapper - return inner \ No newline at end of file + return inner + + +if __name__ == '__main__': + @run_in_subprocess() + def problematic_func(): + print("in problematic_func") + raise ValueError('Fake Error!!!') + + problematic_func() diff --git a/fuseimg/data/ops/aug/geometry.py b/fuseimg/data/ops/aug/geometry.py index 732a9daf8..db682bcf1 100644 --- a/fuseimg/data/ops/aug/geometry.py +++ b/fuseimg/data/ops/aug/geometry.py @@ -38,18 +38,26 @@ def __call__(self, sample_dict: NDict, key: str, rotate: float = 0.0, translate: :return: the augmented image """ aug_input = sample_dict[key] - - # verify - if self._verify_arguments: - assert isinstance(aug_input, torch.Tensor), f"Error: OpAugAffine2D expects torch Tensor, got {type(aug_input)}" - assert len(aug_input.shape) in [2, 3], f"Error: OpAugAffine2D expects tensor with 2 or 3 dimensions. got {aug_input.shape}" - # Support for 2D inputs - implicit single channel - if len(aug_input.shape) == 2: - aug_input = aug_input.unsqueeze(dim=0) - remember_to_squeeze = True - else: - remember_to_squeeze = False + aug_tensor = auf_affine_2D(aug_input, translate=translate, scale=scale, flip=flip, shear=shear, channels=channels, verify_arguments=self._verify_arguments) + + sample_dict[key] = aug_tensor + return sample_dict + +def auf_affine_2D(aug_input, rotate: float = 0.0, translate: Tuple[float, float] = (0.0, 0.0), + scale: Tuple[float, float] = 1.0, flip: Tuple[bool, bool] = (False, False), shear: float = 0.0, + channels: Optional[List[int]] = None, verify_arguments:Optional[bool] = True): + # verify + if verify_arguments: + assert isinstance(aug_input, torch.Tensor), f"Error: OpAugAffine2D expects torch Tensor, got {type(aug_input)}" + assert len(aug_input.shape) in [2, 3], f"Error: OpAugAffine2D expects tensor with 2 or 3 dimensions. got {aug_input.shape}" + + # Support for 2D inputs - implicit single channel + if len(aug_input.shape) == 2: + aug_input = aug_input.unsqueeze(dim=0) + remember_to_squeeze = True + else: + remember_to_squeeze = False # convert to PIL (required by affine augmentation function) if channels is None: @@ -71,12 +79,10 @@ def __call__(self, sample_dict: NDict, key: str, rotate: float = 0.0, translate: # set the augmented channel aug_tensor[channel] = aug_channel_tensor - # squeeze back to 2-dim if needed - if remember_to_squeeze: - aug_tensor = aug_tensor.squeeze(dim=0) - - sample_dict[key] = aug_tensor - return sample_dict + # squeeze back to 2-dim if needed + if remember_to_squeeze: + aug_tensor = aug_tensor.squeeze(dim=0) + return aug_tensor class OpAugCropAndResize2D(OpBase): @@ -156,29 +162,32 @@ def __call__(self, sample_dict: NDict, key: str, axis_squeeze: int) -> NDict: :param axis_squeeze: the axis (1, 2 or 3) to squeeze into channel dimension - typically z axis """ aug_input = sample_dict[key] - - # verify - if self._verify_arguments: - assert isinstance(aug_input, torch.Tensor), f"Error: OpAugSqueeze3Dto2D expects torch Tensor, got {type(aug_input)}" - assert len(aug_input.shape) == 4, f"Error: OpAugSqueeze3Dto2D expects tensor with 4 dimensions. got {aug_input.shape}" - - # aug_input shape is [channels, axis_1, axis_2, axis_3] - if axis_squeeze == 1: - pass - elif axis_squeeze == 2: - aug_input = aug_input.permute((0, 2, 1, 3)) - # aug_input shape is [channels, axis_2, axis_1, axis_3] - elif axis_squeeze == 3: - aug_input = aug_input.permute((0, 3, 1, 2)) - # aug_input shape is [channels, axis_3, axis_1, axis_2] - else: - raise Exception(f"Error: axis squeeze must be 1, 2, or 3, got {axis_squeeze}") - - aug_output = aug_input.reshape((aug_input.shape[0] * aug_input.shape[1],) + aug_input.shape[2:]) - + aug_output =squeeze_3D_to_2D(aug_input, axis_squeeze, self._verify_arguments) + sample_dict[key] = aug_output return sample_dict +def squeeze_3D_to_2D(aug_input, axis_squeeze: int, verify_arguments=True): + # verify + if verify_arguments: + assert isinstance(aug_input, torch.Tensor), f"Error: OpAugSqueeze3Dto2D expects torch Tensor, got {type(aug_input)}" + assert len(aug_input.shape) == 4, f"Error: OpAugSqueeze3Dto2D expects tensor with 4 dimensions. got {aug_input.shape}" + + # aug_input shape is [channels, axis_1, axis_2, axis_3] + if axis_squeeze == 1: + pass + elif axis_squeeze == 2: + aug_input = aug_input.permute((0, 2, 1, 3)) + # aug_input shape is [channels, axis_2, axis_1, axis_3] + elif axis_squeeze == 3: + aug_input = aug_input.permute((0, 3, 1, 2)) + # aug_input shape is [channels, axis_3, axis_1, axis_2] + else: + raise Exception(f"Error: axis squeeze must be 1, 2, or 3, got {axis_squeeze}") + + aug_output = aug_input.reshape((aug_input.shape[0] * aug_input.shape[1],) + aug_input.shape[2:]) + return aug_output + class OpAugUnsqueeze3DFrom2D(OpBase): def __init__(self, verify_arguments: bool = True): """ @@ -197,37 +206,41 @@ def __call__(self, sample_dict: NDict, key: str, axis_squeeze: int, channels: in :param channels: number of channels in the original tensor (before OpAugSqueeze3Dto2D) """ aug_input = sample_dict[key] + aug_output = unsqueeze_3D_from_2D(aug_input, axis_squeeze, channels, self._verify_arguments) - # verify - if self._verify_arguments: - assert isinstance(aug_input, torch.Tensor), f"Error: OpAugUnsqueeze3DFrom2D expects torch Tensor, got {type(aug_input)}" - assert len(aug_input.shape) == 3, f"Error: OpAugUnsqueeze3DFrom2D expects tensor with 3 dimensions. got {aug_input.shape}" - - aug_output = aug_input.reshape((channels, aug_input.shape[0] // channels) + aug_input.shape[1:]) - - if axis_squeeze == 1: - pass - elif axis_squeeze == 2: - # aug_output shape is [channels, axis_2, axis_1, axis_3] - aug_output = aug_output.permute((0, 2, 1, 3)) - # aug_input shape is [channels, axis 1, axis 2, axis 3] - elif axis_squeeze == 3: - # aug_output shape is [channels, axis_3, axis_1, axis_2] - aug_output = aug_output.permute((0, 2, 3, 1)) - # aug_input shape is [channels, axis 1, axis 2, axis 3] - else: - raise Exception(f"Error: axis squeeze must be 1, 2, or 3, got {axis_squeeze}") - sample_dict[key] = aug_output return sample_dict +def unsqueeze_3D_from_2D(aug_input: torch.Tensor, axis_squeeze: int, channels: int, verify_arguments: Optional[bool]=True): + # verify + if verify_arguments: + assert isinstance(aug_input, torch.Tensor), f"Error: OpAugUnsqueeze3DFrom2D expects torch Tensor, got {type(aug_input)}" + assert len(aug_input.shape) == 3, f"Error: OpAugUnsqueeze3DFrom2D expects tensor with 3 dimensions. got {aug_input.shape}" + + + aug_output = aug_input.reshape((channels, aug_input.shape[0] // channels) + aug_input.shape[1:]) + + if axis_squeeze == 1: + pass + elif axis_squeeze == 2: + # aug_output shape is [channels, axis_2, axis_1, axis_3] + aug_output = aug_output.permute((0, 2, 1, 3)) + # aug_input shape is [channels, axis 1, axis 2, axis 3] + elif axis_squeeze == 3: + # aug_output shape is [channels, axis_3, axis_1, axis_2] + aug_output = aug_output.permute((0, 2, 3, 1)) + # aug_input shape is [channels, axis 1, axis 2, axis 3] + else: + raise Exception(f"Error: axis squeeze must be 1, 2, or 3, got {axis_squeeze}") + return aug_output + class OpCrop3D(OpBase): """ crop to certain size. if the image is smaller than the size then its padded. """ def __call__(self, sample_dict: NDict, key: str, output_shape: Tuple[int, int, int], - z_move=0.5,x_move=0.5,y_move=0.5,fill: int = 0, + z_move=0.5,x_move=0.5,y_move=0.5,fill: int = 0, ): """ :param key: key to a tensor stored in sample_dict and get cropped by OpRandomCrop3D @@ -257,10 +270,10 @@ def __call__(self, sample_dict: NDict, key: str, output_shape: Tuple[int, int, i aug_tensor[:depth,:height,:width] = aug_input sample_dict[key] = aug_tensor - + return sample_dict - + class OpResizeTo(OpBase): """ @@ -314,7 +327,7 @@ def get_permutation(self, dim: int, channels_first: bool): if dim < 2: raise Exception(f"Error, dim ({dim}) must be greater or equal to 2.") - + if channels_first: channels = [0] hw = [i for i in range(1, dim)] diff --git a/fuseimg/data/ops/aug/geometry3d.py b/fuseimg/data/ops/aug/geometry3d.py new file mode 100644 index 000000000..9a44fbc7e --- /dev/null +++ b/fuseimg/data/ops/aug/geometry3d.py @@ -0,0 +1,63 @@ +from typing import List, Optional, Tuple, Union + +from torch import Tensor +from PIL import Image + +import numpy +import torch +import torchvision.transforms.functional as TTF +from fuseimg.data.ops.aug import geometry + +from fuse.utils.ndict import NDict + +from fuse.data import OpBase + +class OpRotation3D(OpBase): + """ + 2D affine transformation + """ + + def __init__(self, verify_arguments: bool = True): + """ + :param verify_arguments: this op expects torch tensor with either 2 or 3 dimensions. Set to False to disable verification + """ + super().__init__() + self._verify_arguments = verify_arguments + + def __call__(self, sample_dict: NDict, key: str, ax1_rot: float = 0.0, ax2_rot: float = 0.0, ax3_rot: float = 0) -> Union[None, dict, List[dict]]: + aug_input = sample_dict[key] + + aug_tensor = rotation_in_3d(aug_input, ax1_rot=ax1_rot, ax2_rot=ax2_rot, ax3_rot=ax3_rot) + sample_dict[key] = aug_tensor + return sample_dict + + + +def rotation_in_3d(aug_input: Tensor, ax1_rot: float = 0.0, ax2_rot: float = 0.0, ax3_rot: float = 0): + """ + rotates an input tensor around an axis, when for example z_rot is chosen, + the rotation is in the x-y plane. + Note: rotation angles are in relation to the original axis (not the rotated one) + rotation angles should be given in degrees + :param aug_input:image input should be in shape [channel, ax1, ax2, ax3] + :param ax1_rot: angle to rotate ax2-ax3 plane clockwise + :param ax2_rot: angle to rotate ax3-ax1 plane clockwise + :param ax3_rot: angle to rotate ax1-ax2 plane clockwise + :return: + """ + assert len(aug_input.shape) == 4 # will only work for 3d + channels = aug_input.shape[0] + if ax1_rot != 0: + squeez_img = geometry.squeeze_3D_to_2D(aug_input, axis_squeeze=1) + rot_squeeze = geometry.auf_affine_2D(squeez_img, rotate=ax1_rot) + aug_input = geometry.unsqueeze_3D_from_2D(rot_squeeze, axis_squeeze=1, channels=channels) + if ax2_rot != 0: + squeez_img = geometry.squeeze_3D_to_2D(aug_input, axis_squeeze=2) + rot_squeeze = geometry.auf_affine_2D(squeez_img, rotate=ax2_rot) + aug_input =geometry. unsqueeze_3D_from_2D(rot_squeeze, axis_squeeze=2, channels=channels) + if ax3_rot != 0: + squeez_img = geometry.squeeze_3D_to_2D(aug_input, axis_squeeze=3) + rot_squeeze = geometry.auf_affine_2D(squeez_img, rotate=ax3_rot) + aug_input = geometry.unsqueeze_3D_from_2D(rot_squeeze, axis_squeeze=3, channels=channels) + + return aug_input \ No newline at end of file diff --git a/fuseimg/data/ops/ops_common_imaging.py b/fuseimg/data/ops/ops_common_imaging.py index 1763f6691..c48d45e76 100644 --- a/fuseimg/data/ops/ops_common_imaging.py +++ b/fuseimg/data/ops/ops_common_imaging.py @@ -4,4 +4,7 @@ OpApplyTypesImaging = partial(OpApplyTypes, type_detector = type_detector_imaging, -) \ No newline at end of file +) + + + diff --git a/fuseimg/data/ops/ops_mri.py b/fuseimg/data/ops/ops_mri.py new file mode 100644 index 000000000..a351b1d14 --- /dev/null +++ b/fuseimg/data/ops/ops_mri.py @@ -0,0 +1,1108 @@ +import glob +import os +from typing import Optional +import logging + +import SimpleITK as sitk +import h5py + +import numpy as np +import pydicom +from scipy.ndimage.morphology import binary_dilation + +from fuse.data import OpBase, get_sample_id +from fuse.utils import NDict +from typing import Tuple +import torch +import cv2 +import radiomics + + +class OpExtractDicomsPerSeq(OpBase): + + def __init__(self, seq_ids, series_desc_2_sequence_map, use_order_indicator: bool = False, **kwargs): + super().__init__(**kwargs) + self._seq_ids = seq_ids + self._series_desc_2_sequence_map = series_desc_2_sequence_map + self._use_order_indicator = use_order_indicator + + def __call__(self, sample_dict: NDict, key_in: str, key_out_seq_ids: str, key_out_sequence_prefix: str): + sample_path = sample_dict[key_in] + sample_dict[key_out_seq_ids] = [] + seq_2_info_map = extract_seq_2_info_map(sample_path, self._series_desc_2_sequence_map) + for seq_id in self._seq_ids: + + seq_info_list = seq_2_info_map.get(seq_id, None) + if seq_info_list is None: + # sequence does not exist for the patient + continue + sample_dict[key_out_seq_ids].append(seq_id) + sample_dict[f'{key_out_sequence_prefix}{seq_id}'] = [] + for seq_info in seq_info_list: # could be several sequences/series (sequence/series= path) + + dicom_group_ids, sorted_dicom_groups = sort_dicoms_by_field(seq_info['path'], seq_info['dicom_field'], self._use_order_indicator) + for dicom_group_id, dicom_group in zip(dicom_group_ids, sorted_dicom_groups): + seq_info2 = dict(path=seq_info['path'], + series_num=seq_info['series_num'], + series_desc=seq_info['series_desc'], + dicoms=dicom_group, # each sequence/series path may contain several (sub-)sequence/series + dicoms_id=dicom_group_id + ) + sample_dict[f'{key_out_sequence_prefix}{seq_id}'].append(seq_info2) + + return sample_dict + + +###################################################################### +class OpLoadDicomAsStkVol(OpBase): + ''' + Return location dir of requested sequence + ''' + + def __init__(self, seq_reverse_map=None, **kwargs): + """ + :param reverse_order: sometimes reverse dicoms orders is needed + (for b series in which more than one sequence is provided inside the img_path) + :param is_file: if True loads all dicoms from img_path + :param kwargs: + """ + super().__init__(**kwargs) + if seq_reverse_map is None: + seq_reverse_map = {} + self._seq_reverse_map = seq_reverse_map + + def __call__(self, sample_dict: NDict, key_in_seq_ids: str, key_sequence_prefix: str): + """ + extract_stk_vol loads dicoms into sitk vol + :param img_path: path to dicoms - load all dicoms from this path + :param img_list: list of dicoms to load + :return: list of stk vols + """ + seq_ids = sample_dict[key_in_seq_ids] + + for seq_id in seq_ids: + should_reverse_order = self._seq_reverse_map.get(seq_id, False) + + sequence_info_list = sample_dict[f'{key_sequence_prefix}{seq_id}'] + + for sequence_info in sequence_info_list: + stk_vol = get_stk_volume(img_path=sequence_info['path'], is_file=False, + dicom_files=sequence_info['dicoms'], reverse_order=should_reverse_order) + sequence_info['stk_volume'] = stk_vol + + return sample_dict + + +def get_stk_volume(img_path, is_file, dicom_files, reverse_order): + # load from HDF5 + if img_path[-4::] in 'hdf5': + vol = _read_HDF5_file(img_path) + return vol + + if is_file: + vol = sitk.ReadImage(img_path) + return vol + + series_reader = sitk.ImageSeriesReader() + + if dicom_files is None: + dicom_files = series_reader.GetGDCMSeriesFileNames(img_path) + + if isinstance(dicom_files, str): + dicom_files = [dicom_files] + if img_path not in dicom_files[0]: + dicom_files = [os.path.join(img_path, dicom_file) for dicom_file in dicom_files] + dicom_files = dicom_files[::-1] if reverse_order else dicom_files + series_reader.SetFileNames(dicom_files) + vol = series_reader.Execute() + return vol + + +def _read_HDF5_file(img_path): + with h5py.File(img_path, 'r') as hf: + _array = np.array(hf['array']) + _spacing = hf.attrs['spacing'] + _origin = hf.attrs['origin'] + _world_matrix = np.array(hf.attrs['world_matrix'])[:3, :3] + _world_matrix_unit = _world_matrix / np.linalg.norm(_world_matrix, axis=0) + _world_matrix_unit_flat = _world_matrix_unit.flatten() + + # volume 2 sitk + vol = sitk.GetImageFromArray(_array) + vol.SetOrigin([_origin[i] for i in [1, 2, 0]]) + vol.SetDirection(_world_matrix_unit_flat) + vol.SetSpacing([_spacing[i] for i in [1, 2, 0]]) + return vol + + +############################# +class OpGroupDCESequences(OpBase): + def __init__(self, verbose: bool = True, + **kwargs): + super().__init__(**kwargs) + self._verbose = verbose + + def __call__(self, sample_dict: NDict, key_seq_ids: str, key_sequence_prefix: str): + + """ + extract_list_of_rel_vol extract the volume per seq based on SER_INX_TO_USE + and put in one list + :param vols_dict: dict of sitk vols per seq + :param seq_info: dict of seq description per seq + :return: + """ + + seq_ids = sample_dict[key_seq_ids] + + all_dce_mix_ph_sequences = [f'DCE_mix_ph{i}' for i in range(1, 5)] + ['DCE_mix_ph'] + + existing_dce_mix_ph_sequences = [seq_id for seq_id in all_dce_mix_ph_sequences if seq_id in seq_ids] + # handle multiphase DCE in different series + if existing_dce_mix_ph_sequences: + new_seq_id = 'DCE_mix' + assert new_seq_id not in seq_ids + seq_ids.append(new_seq_id) + sample_dict[f'{key_sequence_prefix}{new_seq_id}'] = [] + for seq_id in existing_dce_mix_ph_sequences: + sequence_info_list = sample_dict[f'{key_sequence_prefix}{seq_id}'] + if seq_id == 'DCE_mix_ph': + series_num_arr = [a['series_num'] for a in sequence_info_list] + inx_sorted = np.argsort(series_num_arr) + sequence_info_list = [sequence_info_list[i] for i in inx_sorted] + sample_dict[f'{key_sequence_prefix}{new_seq_id}'] += sequence_info_list + + delete_seqeunce_from_dict(seq_id=seq_id, sample_dict=sample_dict, + key_sequence_ids=key_seq_ids, key_sequence_prefix=key_sequence_prefix + ) + + return sample_dict + + +############################# +class OpSelectVolumes(OpBase): + def __init__(self, get_indexes_func, selected_seq_ids: list, delete_input_volumes: Optional[bool] = False, verbose: bool = True, + **kwargs): + super().__init__(**kwargs) + self._get_indexes_func = get_indexes_func + self._selected_seq_ids = selected_seq_ids + self._delete_input_volumes = delete_input_volumes + self._verbose = verbose + + def __call__(self, sample_dict: NDict, key_in_seq_ids: str, key_in_sequence_prefix: str, + key_out_volumes: str, key_out_volumes_info: Optional[str] = None + ): + + sample_id = get_sample_id(sample_dict) + seq_ids = sample_dict[key_in_seq_ids] + + sample_dict[f'{key_out_volumes}'] = [] + sample_dict[f'{key_out_volumes_info}'] = [] + + for selected_seq_id in self._selected_seq_ids: + + if selected_seq_id in seq_ids: + sequence_info_list = sample_dict[f'{key_in_sequence_prefix}{selected_seq_id}'] + else: + sequence_info_list = [] + + vol_inx_to_use = _get_as_list(self._get_indexes_func(sample_id, selected_seq_id)) + + for inx in vol_inx_to_use: + + if len(sequence_info_list) == 0: + if len(sample_dict[key_out_volumes]) == 0: + print("============XXXXXXX", sample_id) # todo: write into log file + lgr = logging.getLogger('Fuse') + lgr.info(f'OpSelectVolumes: {sample_id} is excluded', {'attrs': ['bold']}) + return None + seq_volume_template = sample_dict[key_out_volumes][0] + stk_volume = get_zeros_vol(seq_volume_template) + selected_sequence_info = {} + else: + if inx >= len(sequence_info_list): + inx = -1 # take the last + selected_sequence_info = sequence_info_list[inx] + + stk_volume = selected_sequence_info['stk_volume'] + + if len(stk_volume) == 0: + seq_volume_template = sample_dict[key_out_volumes][0] + stk_volume = get_zeros_vol(seq_volume_template) + if self._verbose: + print(f'\n - problem with reading {selected_seq_id} volume!') + + sample_dict[key_out_volumes].append(stk_volume) + sample_dict[key_out_volumes_info].append(dict(seq_id=selected_seq_id, + series_desc=selected_sequence_info.get('series_desc', 'NAN'), + path=selected_sequence_info.get('path', 'NAN'), + dicoms_id=selected_sequence_info.get('dicoms_id', 'NAN')) + ) + if self._delete_input_volumes: + for seq_id in seq_ids: # to save space...also when caching sample_dict + for sequence_info in sample_dict[f'{key_in_sequence_prefix}{seq_id}']: + del sequence_info['stk_volume'] + return sample_dict + + +# def store(sample_dict): +# vv = [sitk.GetArrayFromImage(v) for v in sample_dict['data.input.selected_volumes']] +# vv2 = file_io.load_pickle('/tmp/fuse_temp.pkl') +# for i in range(len(vv)): +# # print(vv[i].shape == vv2[i].shape) +# print(vv[i].max(), vv2[i].max(), np.abs(vv[i]-vv2[i]).max()) +############################ + +class OpResampleStkVolsBasedRef(OpBase): + def __init__(self, reference_inx: int, interpolation: str, **kwargs): + super().__init__(**kwargs) + assert reference_inx is not None # todo: redundant?? + self.reference_inx = reference_inx + self.interpolation = interpolation + + def __call__(self, sample_dict: NDict, + key: str): + + # ------------------------ + # create resampling operator based on ref vol + volumes = sample_dict[key] + assert len(volumes) > 0 + # if self.reference_inx > 0: + # volumes = [volumes[self.reference_inx]]+ volumes[:self.reference_inx]+ volumes[volumes+1:] + + seq_volumes_resampled = [sitk.Cast(v, sitk.sitkFloat32) for v in volumes] + ref_volume = volumes[self.reference_inx] + + resample = self.create_resample(ref_volume, self.interpolation, size=ref_volume.GetSize(), + spacing=ref_volume.GetSpacing()) + + for i in range(len(seq_volumes_resampled)): + if i == self.reference_inx: + continue + + seq_volumes_resampled[i] = resample.Execute(seq_volumes_resampled[i]) + + sample_dict[key] = seq_volumes_resampled + return sample_dict + + def create_resample(self, vol_ref: sitk.sitkFloat32, interpolation: str, size: Tuple[int, int, int], + spacing: Tuple[float, float, float]): + """ + create_resample create resample operator + :param vol_ref: sitk vol to use as a ref + :param interpolation:['linear','nn','bspline'] + :param size: in pixels () + :param spacing: in mm () + :return: resample sitk operator + """ + + if interpolation == 'linear': + interpolator = sitk.sitkLinear + elif interpolation == 'nn': + interpolator = sitk.sitkNearestNeighbor + elif interpolation == 'bspline': + interpolator = sitk.sitkBSpline + + resample = sitk.ResampleImageFilter() + resample.SetReferenceImage(vol_ref) + resample.SetOutputSpacing(spacing) + resample.SetInterpolator(interpolator) + resample.SetSize(size) + return resample + + +####### + +class OpStackList4DStk(OpBase): + def __init__(self, delete_input_volumes: Optional[bool] = False, reference_inx: Optional[int] = 0, **kwargs): + super().__init__(**kwargs) + self._reference_inx = reference_inx + self._delete_input_volumes = delete_input_volumes + + def __call__(self, sample_dict: NDict, key_in: str, key_out_volume4d: str, key_out_ref_volume: str): + vols_stk_list = sample_dict[key_in] + if self._delete_input_volumes: + del sample_dict[key_in] + + vol_arr = [sitk.GetArrayFromImage(vol) for vol in vols_stk_list] + vol_final = np.stack(vol_arr, axis=-1) + vol_final_sitk = sitk.GetImageFromArray(vol_final, isVector=True) + vol_final_sitk.CopyInformation(vols_stk_list[self._reference_inx]) + + sample_dict[key_out_volume4d] = vol_final_sitk + sample_dict[key_out_ref_volume] = vols_stk_list[self._reference_inx] + return sample_dict + + +class OpRescale4DStk(OpBase): + def __init__(self, thres: Optional[tuple] = (1.0, 99.0), method: Optional[str] = 'noclip', **kwargs): + super().__init__(**kwargs) + # self._mask_ch_inx = mask_ch_inx + self._thres = thres + self._method = method + + def __call__(self, sample_dict: NDict, key: str): + stk_vol_4D = sample_dict[key] + + vol_backup = sitk.Image(stk_vol_4D) + vol_array = sitk.GetArrayFromImage(stk_vol_4D) + if len(vol_array.shape) < 4: + vol_array = vol_array[:, :, :, np.newaxis] + # vol_array_pre_rescale = vol_array.copy() + vol_array = apply_rescaling(vol_array, thres=self._thres, method=self._method) + + # if self._mask_ch_inx: + # bool_mask = np.zeros(vol_array_pre_rescale[:, :, :, self._mask_ch_inx].shape) + # bool_mask[vol_array_pre_rescale[:, :, :, self._mask_ch_inx] > 0.3] = 1 + # vol_array[:, :, :, self._mask_ch_inx] = bool_mask + + vol_final = sitk.GetImageFromArray(vol_array) # , isVector=True) + vol_final.CopyInformation(vol_backup) + vol_final = sitk.Image(vol_final) + sample_dict[key] = vol_final + return sample_dict + + +class OpAddMaskFromBoundingBoxAsLastChannel(OpBase): + def __init__(self, name_suffix: Optional[str] = '', **kwargs): + super().__init__(**kwargs) + self._name_suffix = name_suffix + + def __call__(self, sample_dict: NDict, key_in_patch_annotations: str, key_in_ref_volume: str, key_volume4D: str): + + vol_ref = sample_dict[key_in_ref_volume] + patch_annotations = sample_dict[key_in_patch_annotations] + vol_4D_stk = sample_dict[key_volume4D] + + vol_4D_arr = sitk.GetArrayFromImage(vol_4D_stk) + if len(vol_4D_arr.shape) == 3: + vol_4D_arr = np.expand_dims(vol_4D_arr, axis=3) + assert len(vol_4D_arr.shape) == 4 + + bbox_coords = patch_annotations[f'bbox{self._name_suffix}'] + if isinstance(bbox_coords, str): + bbox_coords = np.fromstring(bbox_coords[1:-1], dtype=np.int32, sep=',') + else: + bbox_coords = np.asarray(bbox_coords) + mask_3D_arr = extract_mask_from_annotation(vol_ref, bbox_coords) + mask_4D_arr = np.expand_dims(mask_3D_arr, axis=3) + vol_4D_arr_new = np.concatenate((vol_4D_arr, mask_4D_arr), axis=3) + vol_4D_new_stk = sitk.GetImageFromArray(vol_4D_arr_new) + sample_dict[key_volume4D] = vol_4D_new_stk + + return sample_dict + + +class OpCreatePatchVolumes(OpBase): + def __init__(self, lsn_shape, lsn_spacing, pos_key: str = None, name_suffix: Optional[str] = '', delete_input_volumes=False, + crop_based_annotation=True, **kwargs): + super().__init__(**kwargs) + self._lsn_shape = lsn_shape + self._lsn_spacing = lsn_spacing + self._name_suffix = name_suffix + self._delete_input_volumes = delete_input_volumes + self._crop_based_annotation = crop_based_annotation + if pos_key is None: + self._pos_key = f'centroid{name_suffix}' + else: + self._pos_key = pos_key + + def __call__(self, sample_dict: NDict, key_in_volume4D: str, key_in_ref_volume: str, key_in_patch_annotations: str, + key_out: str): + + vol_ref = sample_dict[key_in_ref_volume] + vol_4D = sample_dict[key_in_volume4D] + patch_annotations = sample_dict[key_in_patch_annotations] + + # read original position + pos_orig = patch_annotations[self._pos_key] + if isinstance(pos_orig, str): + if pos_orig[0] == '(': + pos_orig = pos_orig[1:-1] + if ',' in pos_orig: + sep = ',' + else: + sep = ' ' + pos_orig = np.fromstring(pos_orig, dtype=np.float32, sep=sep) + else: + pos_orig = np.asarray(pos_orig) + + # transform to pixel coordinate in ref coords + pos_vol = np.array(vol_ref.TransformPhysicalPointToContinuousIndex(pos_orig.astype(np.float64))) + + cropped_vol_size = (self._lsn_shape[2], self._lsn_shape[1], self._lsn_shape[0]) + spacing = (self._lsn_spacing[2], self._lsn_spacing[1], self._lsn_spacing[0]) + if self._crop_based_annotation: + vol_cropped = crop_lesion_vol_mask_based(vol_4D, pos_vol, vol_ref, + size=cropped_vol_size, + spacing=spacing, + mask_inx=-1, + is_use_mask=True) + else: + vol_cropped = crop_lesion_vol(vol_4D, pos_vol, vol_ref, + center_slice=pos_vol[2], + size=cropped_vol_size, + spacing=spacing) + + vol_cropped_arr = sitk.GetArrayFromImage(vol_cropped) + if len(vol_cropped_arr.shape) < 4: + # fix dimensions in case of one seq + vol_cropped_arr = vol_cropped_arr[:, :, :, np.newaxis] + + vol_cropped_arr = np.moveaxis(vol_cropped_arr, 3, 0) # move last dimension (sequences / mask) to be first + + assert not np.isnan(vol_cropped_arr).any() # todo: need to revisit for cases with nans (currently there are none) + + sample_dict[key_out] = vol_cropped_arr + + if self._delete_input_volumes: + del sample_dict[key_in_volume4D] + del sample_dict[key_in_ref_volume] + return sample_dict + + +####### + +class OpStk2Dict(OpBase): + def __call__(self, sample_dict: NDict, keys: list): + for key in keys: + vol_stk2 = sample_dict[key] + d = dict(arr=sitk.GetArrayFromImage(vol_stk2), + origin=vol_stk2.GetOrigin(), spacing=vol_stk2.GetSpacing(), + direction=vol_stk2.GetDirection()) + sample_dict[key] = d + + return sample_dict + + +class OpDict2Stk(OpBase): + def __call__(self, sample_dict: NDict, keys: list): + for key in keys: + d = sample_dict[key] + vol_stk = sitk.GetImageFromArray(d['arr']) + vol_stk.SetOrigin(d['origin']) + vol_stk.SetSpacing(d['spacing']) + vol_stk.SetDirection(d['direction']) + sample_dict[key] = vol_stk + + return sample_dict + + +class OpFixProstateBSequence(OpBase): + + def __call__(self, sample_dict: NDict, op_id: Optional[str], + key_sequence_ids: str, key_path_prefix: str, key_in_volumes_prefix: str): + seq_ids = sample_dict[key_sequence_ids] + if 'b_mix' in seq_ids: + + B_SER_FIX = ['diffusie-3Scan-4bval_fs', + 'ep2d_DIFF_tra_b50_500_800_1400_alle_spoelen', + 'diff tra b 50 500 800 WIP511b alle spoelen'] + + def get_single_item(a): + if isinstance(a, list): + assert len(a) == 1 + return a[0] + return a + + b_path = get_single_item(sample_dict[f'{key_path_prefix}b']) + + if os.path.basename(b_path) in B_SER_FIX: + adc_volume = get_single_item(sample_dict[f'{key_in_volumes_prefix}ADC']) + + for b_seq_id in ['b800', 'b400']: + volume = get_single_item(sample_dict[f'{key_in_volumes_prefix}{b_seq_id}']) + + volume.CopyInformation(adc_volume) + return sample_dict + + +######################################3 + + +class OpDeleteSequences(OpBase): + def __init__(self, sequences_to_delete, **kwargs): + super().__init__(**kwargs) + self._sequences_to_delete = sequences_to_delete + + def __call__(self, sample_dict: NDict, op_id: Optional[str], key_sequence_ids): + for seq_id in self._sequences_to_delete: + delete_seqeunce_from_dict(seq_id=seq_id, sample_dict=sample_dict, key_sequence_ids=key_sequence_ids) + + +def delete_seqeunce_from_dict(seq_id, sample_dict, key_sequence_ids, key_sequence_prefix): + seq_ids = sample_dict[key_sequence_ids] + if seq_id in seq_ids: + seq_ids.remove(seq_id) + del sample_dict[f'{key_sequence_prefix}{seq_id}'] + + +def rename_seqeunce_from_dict(sample_dict, seq_id_old, seq_id_new, key_sequence_prefix, key_seq_ids): + seq_ids = sample_dict[key_seq_ids] + if seq_id_old in seq_ids: + assert seq_id_new not in seq_ids + sample_dict[f'{key_sequence_prefix}{seq_id_new}'] = sample_dict[f'{key_sequence_prefix}{seq_id_old}'] + del sample_dict[f'{key_sequence_prefix}{seq_id_old}'] + seq_ids.remove(seq_id_old) + seq_ids.append(seq_id_new) + + return sample_dict + + +############################ + +def get_zeros_vol(vol): + if vol.GetNumberOfComponentsPerPixel() > 1: + ref_zeros_vol = sitk.VectorIndexSelectionCast(vol, 0) + else: + ref_zeros_vol = vol + zeros_vol = np.zeros_like(sitk.GetArrayFromImage(ref_zeros_vol)) + zeros_vol = sitk.GetImageFromArray(zeros_vol) + zeros_vol.CopyInformation(ref_zeros_vol) + return zeros_vol + + +def crop_lesion_vol_mask_based(vol: sitk.sitkFloat32, position: tuple, ref: sitk.sitkFloat32, size: Tuple[int, int, int] = (160, 160, 32), + spacing: Tuple[int, int, int] = (1, 1, 3), margin: Tuple[int, int, int] = (20, 20, 0), + mask_inx=-1, is_use_mask=True): + """ + crop_lesion_vol crop tensor around position + :param vol: vol to crop + :param position: point to crop around + :param ref: reference volume + :param size: size in pixels to crop + :param spacing: spacing to resample the col + :param center_slice: z coordinates of position + :param mask_inx: channel index in which mask is located default: last channel + :param is_use_mask: use mask to define crop bounding box + :return: cropped volume + """ + + vol_np = sitk.GetArrayFromImage(vol) + if is_use_mask: + + mask = sitk.GetArrayFromImage(vol)[:, :, :, mask_inx] + assert set(np.unique(mask)) <= {0, 1} + mask = mask.astype(int) + mask_final = sitk.GetImageFromArray(mask) + mask_final.CopyInformation(ref) + + lsif = sitk.LabelShapeStatisticsImageFilter() + lsif.Execute(mask_final) + bounding_box = np.array(lsif.GetBoundingBox(1)) + vol_np[:, :, :, mask_inx] = mask + else: + bounding_box = np.array([int(position[0]) - int(size[0] / 2), + int(position[1]) - int(size[1] / 2), + int(position[2]) - int(size[2] / 2), + size[0], + size[1], + size[2] + ]) + # in z use a fixed number of slices,based on position + bounding_box[-1] = size[2] + bounding_box[2] = int(position[2]) - int(size[2] / 2) + + bounding_box_size = bounding_box[3:5][np.argmax(bounding_box[3:5])] + dshift = bounding_box[3:5] - bounding_box_size + dshift = np.append(dshift, 0) + + ijk_min_bound = np.maximum(bounding_box[0:3] + dshift - margin, 0) + ijk_max_bound = np.maximum(bounding_box[0:3] + dshift + [bounding_box_size, bounding_box_size, bounding_box[-1]] + margin, 0) + + vol_np_cropped = vol_np[ijk_min_bound[2]:ijk_max_bound[2], ijk_min_bound[1]:ijk_max_bound[1], ijk_min_bound[0]:ijk_max_bound[0], :] + vol_np_resized = np.zeros((size[2], size[0], size[1], vol_np_cropped.shape[-1])) + for si in range(vol_np_cropped.shape[0]): + for ci in range(vol_np_cropped.shape[-1]): + vol_np_resized[si, :, :, ci] = cv2.resize(vol_np_cropped[si, :, :, ci], (size[0], size[1]), interpolation=cv2.INTER_AREA) + + img = sitk.GetImageFromArray(vol_np_resized) + return img + + +def crop_lesion_vol(vol: sitk.sitkFloat32, position: Tuple[float, float, float], ref: sitk.sitkFloat32, size: Tuple[int, int, int] = (160, 160, 32), + spacing: Tuple[int, int, int] = (1, 1, 3), center_slice=None): + """ + crop_lesion_vol crop tensor around position + :param vol: vol to crop + :param position: point to crop around + :param ref: reference volume + :param size: size in pixels to crop + :param spacing: spacing to resample the col + :param center_slice: z coordinates of position + :return: cropped volume + """ + + def get_lesion_mask(position, ref): + mask = np.zeros_like(sitk.GetArrayViewFromImage(ref), dtype=np.uint8) + + coords = np.round(position[::-1]).astype(np.int) + mask[coords[0], coords[1], coords[2]] = 1 + mask = binary_dilation(mask, np.ones((3, 5, 5))) + 0 + mask_sitk = sitk.GetImageFromArray(mask) + mask_sitk.CopyInformation(ref) + + return mask_sitk + + def create_resample(vol_ref: sitk.sitkFloat32, interpolation: str, size: Tuple[int, int, int], + spacing: Tuple[float, float, float]): + """ + create_resample create resample operator + :param vol_ref: sitk vol to use as a ref + :param interpolation:['linear','nn','bspline'] + :param size: in pixels () + :param spacing: in mm () + :return: resample sitk operator + """ + + if interpolation == 'linear': + interpolator = sitk.sitkLinear + elif interpolation == 'nn': + interpolator = sitk.sitkNearestNeighbor + elif interpolation == 'bspline': + interpolator = sitk.sitkBSpline + + resample = sitk.ResampleImageFilter() + resample.SetReferenceImage(vol_ref) + resample.SetOutputSpacing(spacing) + resample.SetInterpolator(interpolator) + resample.SetSize(size) + return resample + + def apply_resampling(img: sitk.sitkFloat32, mask: sitk.sitkFloat32, + spacing: Tuple[float, float, float] = (0.5, 0.5, 3), size: Tuple[int, int, int] = (160, 160, 32), + transform: sitk = None, interpolation: str = 'bspline', + label_interpolator: sitk = sitk.sitkLabelGaussian, + ): + + ref = img if img != [] else mask + size = [int(s) for s in size] + resample = create_resample(ref, interpolation, size=size, spacing=spacing) + + if ~(transform is None): + resample.SetTransform(transform) + img_r = resample.Execute(img) + + resample.SetInterpolator(label_interpolator) + mask_r = resample.Execute(mask) + + return img_r, mask_r + + mask = get_lesion_mask(position, ref) + + vol.SetOrigin((0,) * 3) + mask.SetOrigin((0,) * 3) + vol.SetDirection(np.eye(3).flatten()) + mask.SetDirection(np.eye(3).flatten()) + + ma_centroid = mask > 0.5 + label_analysis_filer = sitk.LabelShapeStatisticsImageFilter() + label_analysis_filer.Execute(ma_centroid) + centroid = label_analysis_filer.GetCentroid(1) + offset_correction = np.array(size) * np.array(spacing) / 2 + corrected_centroid = np.array(centroid) + corrected_centroid[2] = center_slice * np.array(spacing[2]) + offset = corrected_centroid - np.array(offset_correction) + + translation = sitk.TranslationTransform(3, offset) + img, mask = apply_resampling(vol, mask, spacing=spacing, size=size, transform=translation) + + return img + + +def extract_mask_from_annotation(vol_ref, bbox_coords): + xstart = bbox_coords[0] + ystart = bbox_coords[1] + zstart = bbox_coords[2] + xsize = bbox_coords[3] + ysize = bbox_coords[4] + zsize = bbox_coords[5] + + mask = get_zeros_vol(vol_ref) + mask_np = sitk.GetArrayFromImage(mask) + mask_np[zstart:zstart + zsize, ystart:ystart + ysize, xstart:xstart + xsize] = 1.0 + return mask_np + + +def apply_rescaling(img: np.array, thres: tuple = (1.0, 99.0), method: str = 'noclip'): + """ + apply_rescaling rescale each channal using method + :param img: + :param thres: + :param method: + :return: + """ + eps = 0.000001 + + def rescale_single_channel_image(img): + # Deal with negative values first + min_value = np.min(img) + if min_value < 0: + img -= min_value + if method == 'clip': + val_l, val_h = np.percentile(img, thres) + img2 = img + img2[img < val_l] = val_l + img2[img > val_h] = val_h + img2 = (img2.astype(np.float32) - val_l) / (val_h - val_l + eps) + elif method == 'mean': + img2 = img / max(np.mean(img), 1) + elif method == 'median': + img2 = img / max(np.median(img), 1) + # write as op + ###################### + elif method == 'noclip': + val_l, val_h = np.percentile(img, thres) + img2 = img + img2 = (img2.astype(np.float32) - val_l) / (val_h - val_l + eps) + else: + img2 = img + return img2 + + # fix outlier image values + img[np.isnan(img)] = 0 + # Process each channel independently + if len(img.shape) == 4: + for i in range(img.shape[-1]): + img[..., i] = rescale_single_channel_image(img[..., i]) + else: + img = rescale_single_channel_image(img) + + return img + + +def extract_seq_2_info_map(sample_path, series_desc_2_sequence_map): + seq_info_dict = {} + for seq_dir in os.listdir(sample_path): + seq_path = os.path.join(sample_path, seq_dir) + # read series description from dcm files + dcm_files = glob.glob(os.path.join(seq_path, '*.dcm')) + dcm_ds = pydicom.dcmread(dcm_files[0]) + + series_desc = pydicom.dcmread(dcm_files[0]).SeriesDescription + seq_id = series_desc_2_sequence_map.get(series_desc, 'UNKNOWN') + + series_num = extract_ser_num(dcm_ds) + dicom_field = extract_dicom_field(dcm_ds, seq_id) + + seq_info_dict.setdefault(seq_id, []).append(dict(path=seq_path, series_num=series_num, dicom_field=dicom_field, + series_desc=series_desc)) + + return seq_info_dict + + +def extract_ser_num(dcm_ds): + # series number + if hasattr(dcm_ds, 'AcquisitionNumber'): + return int(dcm_ds.AcquisitionNumber) + return int(dcm_ds.SeriesNumber) + + +def extract_dicom_field(dcm_ds, seq_desc): + # dicom key + if seq_desc in ('b_mix', 'b'): + if 'DiffusionBValue' in dcm_ds: + dicom_field = (0x0018, 0x9087) # 'DiffusionBValue' + else: + dicom_field = (0x19, 0x100c) # simens bval tag 0x19 0x100c + elif 'DCE' in seq_desc: + if 'TemporalPositionIdentifier' in dcm_ds: + dicom_field = (0x0020, 0x0100) # Temporal Position Identifier + elif 'TemporalPositionIndex' in dcm_ds: + dicom_field = (0x0020, 0x9128) # Temporal Position Index + else: + dicom_field = (0x0020, 0x0012) # Acqusition Number + elif seq_desc == 'MASK': + dicom_field = (0x0020, 0x0011) # series number + else: + dicom_field = None + + return dicom_field + + +def sort_dicoms_by_field(seq_path, dicom_field, use_order_indicator): + ''' + Return location dir of requested sequence + ''' + + """ + sort_dicom_by_dicom_field sorts the dcm_files based on dicom_field + For some MRI sequences different kinds of MRI series are mixed together (as in bWI) case + This function creates a dict={dicom_field_type:list of relevant dicoms}, + than concats all to a list of the different series types + :param dcm_files: list of all dicoms , mixed + :param dicom_field: dicom field to sort based on + :return: sorted_names_list, list of sorted dicom series + """ + dcm_files = glob.glob(os.path.join(seq_path, '*.dcm')) + dcm_values = {} + dcm_patient_z = {} + dcm_instance = {} + + dcm_ds_list = [pydicom.dcmread(dcm) for dcm in dcm_files] + n_unique_z_image_position_patient = len(np.unique([dcm_ds.ImagePositionPatient[2] for dcm_ds in dcm_ds_list])) + for index, (dcm, dcm_ds) in enumerate(zip(dcm_files, dcm_ds_list)): + patient_z = int(dcm_ds.ImagePositionPatient[2]) + instance_num = int(dcm_ds.InstanceNumber) + if dicom_field is not None: + val = int(dcm_ds[dicom_field].value) + else: + # sort by + val = int(np.floor((instance_num - 1) / n_unique_z_image_position_patient)) + + if val not in dcm_values: + dcm_values[val] = [] + dcm_patient_z[val] = [] + dcm_instance[val] = [] + dcm_values[val].append(os.path.split(dcm)[-1]) + dcm_patient_z[val].append(patient_z) + dcm_instance[val].append(instance_num) + + # ex-sort sub-sequences + sorted_keys = np.sort(list(dcm_values.keys())) + sorted_names_list = [dcm_values[key] for key in sorted_keys] + dcm_patient_z_list = [dcm_patient_z[key] for key in sorted_keys] + dcm_instance_list = [dcm_instance[key] for key in sorted_keys] + + # in-sort each sub-sequence + if use_order_indicator: + # sort from low patient z to high patient z + sorted_names_list_ = [list(np.array(list_of_names)[np.argsort(list_of_z)]) for list_of_names, list_of_z in + zip(sorted_names_list, dcm_patient_z_list)] + else: + # sort by instance number + sorted_names_list_ = [list(np.array(list_of_names)[np.argsort(list_of_z)]) for list_of_names, list_of_z in + zip(sorted_names_list, dcm_instance_list)] + + return sorted_keys, sorted_names_list_ + + +def _get_as_list(x): + if isinstance(x, list): + return x + return [x] + + +############################ +class OpExtractLesionPropFromBBoxAnotation(OpBase): + def __init__(self, get_annotations_func, **kwargs): + super().__init__(**kwargs) + self._get_annotations_func = get_annotations_func + + def __call__(self, sample_dict: NDict, key_in_ref_volume: str, key_out_lesion_prop: str, key_out_cols: str): + sample_id = get_sample_id(sample_dict) + annotations_df = self._get_annotations_func(sample_id) + + vol_ref = sample_dict[key_in_ref_volume] + sample_dict[key_out_lesion_prop] = [] + + bbox_coords = ((annotations_df[annotations_df['Patient ID'] == sample_id]['Start Column'].values[0], + annotations_df[annotations_df['Patient ID'] == sample_id]['Start Row'].values[0]), + (annotations_df[annotations_df['Patient ID'] == sample_id]['End Column'].values[0], + annotations_df[annotations_df['Patient ID'] == sample_id]['End Row'].values[0])) + + start_slice = annotations_df[annotations_df['Patient ID'] == sample_id]['Start Slice'].values[0] + end_slice = annotations_df[annotations_df['Patient ID'] == sample_id]['End Slice'].values[0] + lesion_prop, cols = extarct_lesion_prop_from_annotation(vol_ref, bbox_coords, start_slice, + end_slice) + + sample_dict[key_out_lesion_prop] = lesion_prop + sample_dict[key_out_cols] = cols + + return sample_dict + + +class OpExtractPatchAnotations(OpBase): + + def __call__(self, sample_dict: NDict, key_in_ref_volume: str, key_in_annotations, + key_out: str): + vol_ref = sample_dict[key_in_ref_volume] + annotations = sample_dict[key_in_annotations] + + bbox_coords = ((annotations['Start Column'], + annotations['Start Row']), + (annotations['End Column'], + annotations['End Row'])) + + start_slice = annotations['Start Slice'] + end_slice = annotations['End Slice'] + lesion_prop, cols = extarct_lesion_prop_from_annotation(vol_ref, bbox_coords, start_slice, + end_slice) + + sample_dict[key_out] = dict(zip(cols, lesion_prop[0])) + + return sample_dict + + +def extarct_lesion_prop_from_mask(mask, T_connecetd_component_dist=40, minimumObjectSize=3): + mask_bool = mask > 0 + + dist_img = sitk.SignedMaurerDistanceMap(mask_bool, insideIsPositive=False, squaredDistance=False, useImageSpacing=False) + seeds = sitk.ConnectedComponent(dist_img < T_connecetd_component_dist) + seeds = sitk.RelabelComponent(seeds, minimumObjectSize=minimumObjectSize) + ws = sitk.MorphologicalWatershedFromMarkers(dist_img, seeds, markWatershedLine=True) # slow... + ws = sitk.Mask(ws, sitk.Cast(mask_bool, ws.GetPixelID())) + + shape_stats = sitk.LabelShapeStatisticsImageFilter() + shape_stats.ComputeOrientedBoundingBoxOn() + shape_stats.Execute(ws) + stats_list = [(shape_stats.GetCentroid(i), + shape_stats.GetBoundingBox(i), + shape_stats.GetPhysicalSize(i), + shape_stats.GetElongation(i), + shape_stats.GetOrientedBoundingBoxSize(i)[0], + shape_stats.GetOrientedBoundingBoxSize(i)[1], + shape_stats.GetOrientedBoundingBoxSize(i)[2], + shape_stats.GetRoundness(i), + max(shape_stats.GetEquivalentEllipsoidDiameter(i)), + shape_stats.GetFlatness(i) + ) + for i in shape_stats.GetLabels()] + + cols = ["centroid", "bbox", "volume", "elongation", "size_bbox_x", "size_bbox_y", "size_bbox_z", + "roudness", "longest_elip_diam", "flateness"] + return stats_list, cols + + +def extarct_lesion_prop_from_annotation(vol_ref, bbox_coords, start_slice, end_slice): + mask = get_zeros_vol(vol_ref) + mask_np = sitk.GetArrayFromImage(mask) + mask_np[start_slice:end_slice, bbox_coords[0][1]:bbox_coords[1][1], bbox_coords[0][0]:bbox_coords[1][0]] = 1.0 + mask_final = sitk.GetImageFromArray(mask_np) + mask_final.CopyInformation(vol_ref) + mask_final = sitk.Image(mask_final) + return extarct_lesion_prop_from_mask(mask_final) + + +############################ +# radiomics operator + +class OpReadSTKImage(OpBase): + def __init__(self, seq_id, get_image_file, **kwargs): + super().__init__(**kwargs) + self._seq_id = seq_id + self._get_image_file = get_image_file + + def __call__(self, sample_dict: NDict, key_sequence_prefix: str, key_seq_ids: str): + sample_id = get_sample_id(sample_dict) + img_file = self._get_image_file(sample_id) + vol = sitk.ReadImage(img_file) + sample_dict[key_seq_ids].append(self._seq_id) + sample_dict[f'{key_sequence_prefix}{self._seq_id}'] = [dict(path=img_file, + stk_volume=vol)] + return sample_dict + + +class OpExtractRadiomics(OpBase): + def __init__(self, extractor, setting, **kwargs): + super().__init__(**kwargs) + self.seq_inx_list = setting['seq_inx_list'] + self.seq_list = setting['seq_list'] + self.setting = setting + self.extractor = extractor + + def __call__(self, sample_dict: NDict, key_in_vol_4d: str, key_out_radiomics_results: str): + + vol = sitk.GetArrayFromImage(sample_dict[key_in_vol_4d]) + # fix vol to shape of tensor volume + vol_np = np.moveaxis(vol, 3, 0) + + sample_dict[key_out_radiomics_results] = [] + + maskPath = get_maskpath(vol_np, mask_inx=-1, mask_type=self.setting['maskType']) + result_all = {} + for seq_inx, seq in zip(self.seq_inx_list, self.seq_list): + imagePath = get_imagepath(vol_np, seq_inx) + if self.setting['norm_method'] != 'default': + imagePath = norm_volume(vol_np, seq_inx, imagePath, maskPath, self.setting, normMethod=self.setting['norm_method']) + + result = self.extractor.execute(imagePath, maskPath) + keys_ = list(result.keys()) + for key in keys_: + new_key = key + '_seq=' + seq + '_' + self.setting['maskType'] + result[new_key] = result.pop(key) + result_all.update(result) + + if self.setting['applyLog']: + sigmaValues = np.arange(5., 0., -.5)[::1] + for logImage, imageTypeName, inputKwargs in radiomics.imageoperations.getLoGImage(imagePath, maskPath, + sigma=sigmaValues): + logFirstorderFeatures = radiomics.firstorder.RadiomicsFirstOrder(logImage, maskPath, **inputKwargs) + logFirstorderFeatures.enableAllFeatures() + result = logFirstorderFeatures.execute() + keys_ = list(result.keys()) + for key in keys_: + new_key = key + '_seq=' + seq + '_' + self.setting['maskType'] + result[new_key] = result.pop(key) + result_all.update(result) + + # + # Show FirstOrder features, calculated on a wavelet filtered image + # + if self.setting['applyWavelet']: + for decompositionImage, decompositionName, inputKwargs in radiomics.imageoperations.getWaveletImage(imagePath, + maskPath): + waveletFirstOrderFeaturs = radiomics.firstorder.RadiomicsFirstOrder(decompositionImage, maskPath, **inputKwargs) + waveletFirstOrderFeaturs.enableAllFeatures() + result = waveletFirstOrderFeaturs.execute() + keys_ = list(result.keys()) + for key in keys_: + new_key = key + '_seq=' + seq + '_' + self.setting['maskType'] + result[new_key] = result.pop(key) + result_all.update(result) + + sample_dict[key_out_radiomics_results] = result_all + + return sample_dict + + +def norm_volume(vol_np, seq_inx, imagePath, maskPath, setting, normMethod='default', vol_inx_for_breast_seg=6): + if normMethod == 'default': + return imagePath + + if normMethod == 'tumor_area': + dil_filter = sitk.BinaryDilateImageFilter() + dil_filter.SetKernelRadius(20) + maskPath_binary = sitk.Cast(maskPath, sitk.sitkInt8) + tumor_extand_mask = dil_filter.Execute(maskPath_binary) + firstorder_tmp = radiomics.firstorder.RadiomicsFirstOrder(imagePath, tumor_extand_mask, **setting) + firstorder_tmp.enableFeatureByName('Mean', True) + firstorder_tmp.enableFeatureByName('Variance', True) + results_tmp = firstorder_tmp.execute() + print(results_tmp) + imagePath = (imagePath - results_tmp['Mean']) / np.sqrt(results_tmp['Variance']) + # elif normMethod == 'breast_area': #todo: check installation of FCM!!! + # vol_shape = vol_np.shape + # + # image = vol_np[vol_inx_for_breast_seg, int(vol_shape[1] / 2), :, :] + # image_norm = (image - image.min()) / (image.max() - image.min()) * 256 + # image_rgb = image_norm.astype(np.uint8) + # image_seg = FCM(image=image_rgb, image_bit=8, n_clusters=2, m=2, epsilon=0.05, max_iter=100) # https://github.com/jeongHwarr/various_FCM_segmentation + # image_seg.form_clusters() + # image_seg_res = image_seg.segmentImage() + # breast_label = 1 - image_seg_res[2, 2] + # mask_breast = np.zeros(image_seg_res.shape) + # mask_breast[image_seg_res == breast_label] = 1 + # + # vol_slice = vol_np[seq_inx, int(vol_shape[1] / 2), :, :] + # img_mean = np.mean(vol_slice[mask_breast == 1]) + # img_std = np.std(vol_slice[mask_breast == 1]) + # imagePath = (imagePath - img_mean) / img_std + else: + raise NotImplementedError(normMethod) + + return imagePath + + +def get_maskpath(vol_np, mask_inx=-1, mask_type='full'): + maskPath = sitk.GetImageFromArray(vol_np[mask_inx, :, :, :]) + if mask_type == 'edge': + maskPath_binary = sitk.Cast(maskPath, sitk.sitkInt8) + maskPath_edge = sitk.BinaryDilate(maskPath_binary) - sitk.BinaryErode(maskPath_binary) + maskPath = maskPath_edge + + return maskPath + + +def get_imagepath(vol_np, seq_inx): + imagePath = sitk.GetImageFromArray(vol_np[seq_inx, :, :, :]) + return imagePath diff --git a/fuseimg/datasets/duke.py b/fuseimg/datasets/duke.py new file mode 100644 index 000000000..a6fb22fad --- /dev/null +++ b/fuseimg/datasets/duke.py @@ -0,0 +1,562 @@ +import numpy as np +import glob +import os +import radiomics +from typing import Hashable, Optional, Sequence + +import fuse.data.ops.ops_common +import fuseimg.data.ops.ops_common_imaging +from fuse.data.ops import ops_common +from functools import partial + +from fuseimg.data.ops.aug import geometry, geometry3d +from fuse.data.ops.ops_aug_common import OpSample, OpRandApply +from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform + +import pandas as pd + +from fuse.data import DatasetDefault +from fuse.data import PipelineDefault +from fuse.data.datasets.caching.samples_cacher import SamplesCacher +from fuse.data.ops.op_base import OpReversibleBase, OpBase +from fuse.data.ops import ops_read, ops_cast +from fuse.data.utils.sample import get_sample_id +from fuse.utils import NDict +from fuseimg.data.ops import ops_mri + +import torch + +from fuseimg.datasets.duke_label_type import DukeLabelType + + +def get_selected_series_index(sample_id, seq_id): + patient_id = sample_id[0] + if patient_id in ['Breast_MRI_120', 'Breast_MRI_596']: + map = {'DCE_mix': [2], 'MASK': [0]} + else: + map = {'DCE_mix': [1], 'MASK': [0]} + return map[seq_id] + + +class Duke: + DUKE_DATASET_VER = 0 + + @staticmethod + def dataset(label_type: Optional[DukeLabelType] = None, train: Optional[int] = False, + cache_dir: Optional[str] = None, data_dir: Optional[str] = None, + select_series_func=get_selected_series_index, reset_cache: bool = False, num_workers: int = 10, + sample_ids: Optional[Sequence[Hashable]] = None, verbose: Optional[bool] = True, + cache_kwargs: Optional[dict] = None) -> DatasetDefault: + + """ + :param label_type: type of label to use + :param cache_dir: path to store the cache of the static pipeline + :param data_dir: path to the original data + :param select_series_func: which series to select for DCE_mix sequences + :param reset_cache: + :param num_workers: number of processes used for caching + :param sample_ids: list of selected patient_ids for the dataset + :return: + """ + + if sample_ids is None: + sample_ids = Duke.sample_ids() + + static_pipeline = Duke.static_pipeline(data_dir=data_dir, select_series_func=select_series_func, verbose=verbose) + dynamic_pipeline = Duke.dynamic_pipeline(data_dir=data_dir, train=train, label_type=label_type, verbose=verbose) + + if cache_dir is None: + cacher = None + else: + if cache_kwargs is None: + cache_kwargs = {} + cacher = SamplesCacher(f'duke_cache_ver{Duke.DUKE_DATASET_VER}', + static_pipeline, + [cache_dir], restart_cache=reset_cache, workers=num_workers, + ignore_nan_inequality=True, + **cache_kwargs) + + my_dataset = DatasetDefault(sample_ids=sample_ids, + static_pipeline=static_pipeline, + dynamic_pipeline=dynamic_pipeline, + cacher=cacher + ) + my_dataset.create() + return my_dataset + + @staticmethod + def sample_ids(): + return [f'Breast_MRI_{i:03d}' for i in range(1, 923)] + + @staticmethod + def static_pipeline(select_series_func, data_dir=None, with_rescale: Optional[bool] = True, + output_stk_volumes: Optional[bool] = False, output_patch_volumes: Optional[bool] = True, + verbose: Optional[bool] = True, duke_patch_annotations_df: Optional[pd.DataFrame] = None, name_suffix='') -> PipelineDefault: + + data_dir = Duke.get_data_dir_from_environment_variable() if data_dir is None else data_dir + mri_dir = os.path.join(data_dir, 'manifest-1607053360376') + mri_dir2 = os.path.join(mri_dir, 'Duke-Breast-Cancer-MRI') + metadata_path = os.path.join(mri_dir, 'metadata.csv') + + series_desc_2_sequence_map = get_series_desc_2_sequence_mapping(metadata_path) + seq_ids = ['DCE_mix_ph1', + 'DCE_mix_ph2', + 'DCE_mix_ph3', + 'DCE_mix_ph4', + 'DCE_mix', + 'DCE_mix_ph'] + + static_pipeline_steps = [ + # step 1: map sample_ids to + (OpDukeSampleIDDecode(data_path=mri_dir2), + dict(key_out='data.input.mri_path')), + + # step 2: read sequences + (ops_mri.OpExtractDicomsPerSeq(seq_ids=seq_ids, series_desc_2_sequence_map=series_desc_2_sequence_map, + use_order_indicator=False), + dict(key_in='data.input.mri_path', + key_out_seq_ids='data.input.seq_ids', + key_out_sequence_prefix='data.input.sequence.')), + + # step 3: Load STK volumes of MRI sequences + (ops_mri.OpLoadDicomAsStkVol(), + dict(key_in_seq_ids='data.input.seq_ids', + key_sequence_prefix='data.input.sequence.')), + + # step 4: group DCE sequences into DCE_mix + (ops_mri.OpGroupDCESequences(), + dict(key_seq_ids='data.input.seq_ids', + key_sequence_prefix='data.input.sequence.' + )), + + # step 5: select single volume from DCE_mix sequence + (ops_mri.OpSelectVolumes(get_indexes_func=select_series_func, selected_seq_ids=['DCE_mix'], + delete_input_volumes=True), + dict(key_in_seq_ids='data.input.seq_ids', + key_in_sequence_prefix='data.input.sequence.', + key_out_volumes='data.input.selected_volumes', + key_out_volumes_info='data.input.selected_volumes_info')), + + # step 6: set first volume to be the reference volume and register other volumes with respect to it + (ops_mri.OpResampleStkVolsBasedRef(reference_inx=0, interpolation='bspline'), + dict(key='data.input.selected_volumes')), + + # step 7: create a single 4D volume from all the sequences (4th channel is the sequence) + (ops_mri.OpStackList4DStk(delete_input_volumes=True), dict(key_in='data.input.selected_volumes', + key_out_volume4d='data.input.volume4D', + key_out_ref_volume='data.input.ref_volume')), + + ] + if with_rescale: + # step 8: + static_pipeline_steps += [(ops_mri.OpRescale4DStk(), dict(key='data.input.volume4D'))] + + # step 9: read raw annotations - will be used for labels, features, and also for creating lesion properties + static_pipeline_steps += [(ops_read.OpReadDataframe(data=get_duke_raw_annotations_df(data_dir), key_column='Patient ID'), + dict(key_out_group='data.input.annotations'))] + + # step 10: add lesion properties for each sample id + if output_patch_volumes: + if duke_patch_annotations_df is not None: + # read previousy computed patch annotations + static_pipeline_steps += [ + (ops_read.OpReadDataframe(data=duke_patch_annotations_df, key_column='Patient ID'), + dict(key_out_group='data.input.patch_annotations')), + ] + else: + # generate patch annotations + static_pipeline_steps += [ + # add lesion features + (ops_mri.OpExtractPatchAnotations(), + dict(key_in_ref_volume='data.input.ref_volume', + key_in_annotations='data.input.annotations', + key_out='data.input.patch_annotations')) + ] + + static_pipeline_steps += [ + # step 11: generate a mask from the lesion BB and append as a new (last) channel + (ops_mri.OpAddMaskFromBoundingBoxAsLastChannel(name_suffix=name_suffix), + dict(key_volume4D='data.input.volume4D', + key_in_ref_volume='data.input.ref_volume', + key_in_patch_annotations='data.input.patch_annotations')), + + # step 12: create patch volumes using the mask channel: (i) fixed size (original scale) around center of annotatins (orig), and (ii) entire annotations + (ops_mri.OpCreatePatchVolumes(lsn_shape=(9, 100, 100), lsn_spacing=(1, 0.5, 0.5), + crop_based_annotation=True, + name_suffix=name_suffix, + delete_input_volumes=not output_stk_volumes), + dict(key_in_volume4D='data.input.volume4D', + key_in_ref_volume='data.input.ref_volume', + key_in_patch_annotations='data.input.patch_annotations', + key_out='data.input.patch_volume')) + ] + if output_stk_volumes: + static_pipeline_steps += [ + # step 13: move STK volumes to ndarrays - to allow quick saving to disk + (ops_mri.OpStk2Dict(), + dict(keys=['data.input.volume4D', 'data.input.ref_volume'])) + ] + static_pipeline = PipelineDefault("static", static_pipeline_steps, verbose=verbose) + + return static_pipeline + + @staticmethod + def dynamic_pipeline(data_dir: Optional[str] = None, label_type: Optional[DukeLabelType] = None, + train: Optional[bool] = False, num_channels: Optional[int] = 1, + verbose: Optional[bool] = True, + use_entire_lesion_volume: Optional[bool] = True, + add_clinical_features: Optional[bool] = False): + assert use_entire_lesion_volume + volume_key = 'data.input.patch_volume' if use_entire_lesion_volume else 'data.input.patch_volume_orig' + + def delete_last_channel_in_volume(sample_dict: NDict): + vol = sample_dict[volume_key] + vol = vol[:-1] # remove last channel + sample_dict[volume_key] = vol + return sample_dict + + dynamic_steps = [ + # step 1: delete the mask channel + (fuse.data.ops.ops_common.OpLambda(func=delete_last_channel_in_volume), + dict(key=None)), + + # step 2: turn volume to tensor + (ops_cast.OpToTensor(), dict(key=volume_key, dtype=torch.float32)), + ] + if train: + # step 3: augmentations + dynamic_steps += [ + # step 3.1. 3D rotation + # [ + # ('data.input',), + # rotation_in_3d, + # {'z_rot': Uniform(-5.0, 5.0), 'y_rot': Uniform(-5.0, 5.0), 'x_rot': Uniform(-5.0, 5.0)}, + # {'apply': RandBool(0.5)} + # ], + (OpRandApply(OpSample(geometry3d.OpRotation3D()), 0.5), + dict(key='data.input.patch_volume', + ax1_rot=Uniform(-5.0, 5.0), + ax2_rot=Uniform(-5.0, 5.0), + ax3_rot=Uniform(-5.0, 5.0))), + + # step 3.2.1 3D => 2D + # [ + # ('data.input',), + # squeeze_3d_to_2d, + # {'axis_squeeze': 'z'}, + # {} + # ], + (geometry.OpAugSqueeze3Dto2D(), dict(key='data.input.patch_volume', axis_squeeze=1)), + + # step 3.2.2 2D affine transformation + # [ + # ('data.input',), + # aug_op_affine, + # {'rotate': Uniform(0, 360.0), + # 'translate': (RandInt(-4, 4), RandInt(-4, 4)), + # 'flip': (RandBool(0.5), RandBool(0.5)), + # 'scale': Uniform(0.9, 1.1), + # }, + # {'apply': RandBool(0.5)} + # ], + + (OpRandApply(OpSample(geometry.OpAugAffine2D()), 0.5), dict( + key="data.input.patch_volume", + rotate=Uniform(0, 360.0), + scale=Uniform(0.9, 1.1), + flip=(RandBool(0.5), RandBool(0.5)), + translate=(RandInt(-4, 4), RandInt(-4, 4)) + )), + + # step 3.2.3 2D => 3D + # [ + # ('data.input',), + # unsqueeze_2d_to_3d, + # {'channels': num_channels, 'axis_squeeze': 'z'}, + # {} + # ], + + (geometry.OpAugUnsqueeze3DFrom2D(), dict(key='data.input.patch_volume', + axis_squeeze=1, channels=num_channels, + )), + + ] + + keys_2_keep = [volume_key] + key_ground_truth = 'data.ground_truth' + if add_clinical_features or (label_type is not None): + data_dir = Duke.get_data_dir_from_environment_variable() if data_dir is None else data_dir + + # step 4 (optional): read clinical data + dynamic_steps += [(ops_read.OpReadDataframe(data=get_duke_clinical_data_df(data_dir), + key_column='Patient Information:Patient ID'), + dict(key_out_group='data.input.clinical_data'))] + if label_type is not None: + # dynamic_steps.append((OpAddDukeLabelAndClinicalFeatures(label_type=label_type), + # dict(key_in='data.input.clinical_data', key_out_gt=key_ground_truth, + # key_out_clinical_features='data.clinical_features'))) + # + # step 5: add ground truth label + dynamic_steps.append((ops_common.OpLambda(func=label_type.get_value), + dict(key='data.input.clinical_data', key_out=key_ground_truth))) + + # step 6: remove entries with Nan labels + dynamic_steps.append((ops_common.OpLambda(func=partial(remove_entries_with_nan_label, key=key_ground_truth)), + dict(key=None))) + keys_2_keep.append(key_ground_truth) + + if add_clinical_features: + key_clinical_features = 'data.clinical_features' + dynamic_steps.append((ops_common.OpLambda(func=label_type.select_features), + dict(key='data.input.clinical_data', key_out=key_clinical_features))) + keys_2_keep.append(key_clinical_features) + + for key in keys_2_keep: + if key == volume_key: + continue + dtype = torch.int64 if key == key_ground_truth else torch.float32 + dynamic_steps += [(ops_cast.OpToTensor(), dict(key=key, dtype=dtype))] + + dynamic_steps.append((ops_common.OpKeepKeypaths(), dict(keep_keypaths=keys_2_keep))) + dynamic_pipeline = PipelineDefault("dynamic", dynamic_steps, verbose=verbose) + + return dynamic_pipeline + + @staticmethod + def get_data_dir_from_environment_variable(): + return os.environ["DUKE_DATA_PATH"] + + +class OpDukeSampleIDDecode(OpReversibleBase): + ''' + decodes sample id into path of MRI images + ''' + + def __init__(self, data_path: str, **kwargs): + super().__init__(**kwargs) + self._data_path = data_path + + def __call__(self, sample_dict: NDict, key_out: str, op_id: Optional[str]) -> NDict: + sid = get_sample_id(sample_dict) + + sample_dict[key_out] = get_sample_path(self._data_path, sid) + + return sample_dict + + def reverse(self, sample_dict: dict, key_to_reverse: str, key_to_follow: str, op_id: Optional[str]) -> dict: + return sample_dict + + +class OpAddDukeLabelAndClinicalFeatures(OpBase): + ''' + decodes sample id into path of MRI images + ''' + + def __init__(self, label_type: DukeLabelType, is_concat_features_to_input: Optional[bool] = False, **kwargs): + super().__init__(**kwargs) + self._label_type = label_type + self._is_concat_features_to_input = is_concat_features_to_input + + def __call__(self, sample_dict: NDict, key_in: str, + key_out_gt: str, key_out_clinical_features: str) -> NDict: + clinical_features = sample_dict[key_in] + label_val = self._label_type.get_value(clinical_features) + if np.isnan(label_val): + return None # should filter example (instead of sample_dict['data.filter'] = True ) + + label_tensor = torch.tensor(label_val, dtype=torch.int64) + sample_dict[key_out_gt] = label_tensor # 'data.ground_truth' + + # add clinical + features_to_use = self._label_type.select_features() + + clinical_features_to_use = torch.tensor([float(clinical_features[feature]) for feature in features_to_use], + dtype=torch.float32) + sample_dict[key_out_clinical_features] = clinical_features_to_use # 'data.clinical_features' + + if self._is_concat_features_to_input: + # select input channel + input_tensor = sample_dict['data.input'] + input_shape = input_tensor.shape + for feature in clinical_features_to_use: + input_tensor = torch.cat( + (input_tensor, feature.repeat(input_shape[1], input_shape[2], input_shape[3]).unsqueeze(0)), dim=0) + sample_dict['data.input'] = input_tensor + + return sample_dict + + +def get_duke_raw_annotations_df(duke_data_dir): + annotations_path = os.path.join(duke_data_dir, 'Annotation_Boxes.csv') + annotations_df = pd.read_csv(annotations_path) + return annotations_df + + +def get_duke_clinical_data_df(duke_data_dir): + annotations_path = os.path.join(duke_data_dir, 'Clinical_and_Other_Features.xlsx') + + df = pd.read_excel(annotations_path, sheet_name='Data', nrows=10) + + columns = [] + col_header = '' + for i in range(df.shape[1]): + if not df.columns[i].startswith('Unnamed'): + col_header = df.columns[i] + col_name = col_header.strip() + + for row in [0, 1]: + s = df.iloc[row, i] + if isinstance(s, str) and len(s.strip()) > 0: + col_name += ':' + s.strip() + if col_name not in columns: + break + columns.append(col_name) + + annotations_df = pd.read_excel(annotations_path, sheet_name='Data', skiprows=3, + header=None, names=columns) + return annotations_df + + +def get_samples_for_debug(data_dir, n_pos, n_neg, label_type, sample_ids=None): + annotations_df = get_duke_clinical_data_df(data_dir).set_index('Patient Information:Patient ID') + if sample_ids is not None: + annotations_df = annotations_df.loc[sample_ids] + label_values = label_type.get_value(annotations_df) + patient_ids = annotations_df.index + debug_sample_ids = [] + for label_val, n_vals in zip([True, False], [n_pos, n_neg]): + debug_sample_ids += patient_ids[label_values == label_val].values.tolist()[:n_vals] + return debug_sample_ids + + +def get_series_desc_2_sequence_mapping(metadata_path: str): + # read metadata file and match between series_desc in metadata file and sequence + metadata_df = pd.read_csv(metadata_path) + series_description_list = metadata_df['Series Description'].unique() + + series_desc_2_sequence_mapping = {'ax dyn': 'DCE_mix_ph'} + + patterns = ['1st', '2nd', '3rd', '4th'] + for i_phase in range(1, 5): + seq_id = f'DCE_mix_ph{i_phase}' + phase_patterns = [patterns[i_phase - 1], f'{i_phase}ax', f'{i_phase}Ax', f'{i_phase}/ax', f'{i_phase}/Ax'] + + for series_desc in series_description_list: + has_match = any(p in series_desc for p in phase_patterns) + if has_match: + series_desc2 = series_desc.replace(f'{i_phase}ax', f'{i_phase}/ax').replace(f'{i_phase}Ax', + f'{i_phase}/Ax') + series_desc_2_sequence_mapping[series_desc] = seq_id + series_desc_2_sequence_mapping[series_desc2] = seq_id + + return series_desc_2_sequence_mapping + + +def get_sample_path(data_path, sample_id): + sample_path_pattern = os.path.join(data_path, sample_id, '*') + sample_path = glob.glob(sample_path_pattern) + assert len(sample_path) == 1 + return sample_path[0] + + +################################## +class DukeRadiomics(Duke): + + @staticmethod + def static_pipeline(select_series_func, data_dir: Optional[str] = None, verbose: Optional[bool] = True) -> PipelineDefault: + # remove scaling operator for radiomics calculation + static_pipline = Duke.static_pipeline(data_dir=data_dir, select_series_func=select_series_func, + with_rescale=False, + output_patch_volumes=False, + output_stk_volumes=True, verbose=verbose) + return static_pipline + + @staticmethod + def dynamic_pipeline(radiomics_extractor_setting: dict, + label_type: Optional[DukeLabelType] = None, + verbose: Optional[bool] = False): + + radiomics_extractor = radiomics.featureextractor.RadiomicsFeatureExtractor( + **radiomics_extractor_setting) # todo: tal: move to OpExtractRadiomics + dynamic_steps = [ + (ops_mri.OpDict2Stk(), + dict(keys=['data.input.volume4D', 'data.input.ref_volume'])), + (ops_mri.OpExtractRadiomics(radiomics_extractor, radiomics_extractor_setting), + dict(key_in_vol_4d='data.input.volume4D', key_out_radiomics_results='data.radiomics'))] + + keys_2_keep = ['data.radiomics.original'] + if label_type is not None: + key_ground_truth = 'data.ground_truth' + dynamic_steps.append((OpAddDukeLabelAndClinicalFeatures(label_type=label_type), + dict(key_in='data.input.annotations', key_out_gt=key_ground_truth, + key_out_clinical_features='data.clinical_features'))) + keys_2_keep.append(key_ground_truth) + dynamic_steps.append((ops_common.OpKeepKeypaths(), dict(keep_keypaths=keys_2_keep))) + + dynamic_pipeline = PipelineDefault("dynamic", dynamic_steps, verbose=verbose) + + return dynamic_pipeline + + @staticmethod + def dataset(radiomics_extractor_setting: dict, label_type: Optional[DukeLabelType] = None, cache_dir: Optional[str] = None, + data_dir: Optional[str] = None, + select_series_func=get_selected_series_index, reset_cache: bool = False, num_workers: int = 10, + sample_ids: Optional[Sequence[Hashable]] = None, verbose: Optional[bool] = True, + cache_kwargs: Optional[dict] = None) -> DatasetDefault: + + """ + :param label_type: type of label to use + :param cache_dir: path to store the cache of the static pipeline + :param data_dir: path to the original data + :param select_series_func: which series to select for DCE_mix sequences + :param reset_cache: + :param num_workers: number of processes used for caching + :param sample_ids: list of selected patient_ids for the dataset + :return: + """ + + if sample_ids is None: + sample_ids = DukeRadiomics.sample_ids() + + static_pipeline = DukeRadiomics.static_pipeline(data_dir=data_dir, select_series_func=select_series_func, verbose=verbose) + dynamic_pipeline = DukeRadiomics.dynamic_pipeline(radiomics_extractor_setting=radiomics_extractor_setting, + label_type=label_type, verbose=verbose) + + if cache_dir is None: + cacher = None + else: + if cache_kwargs is None: + cache_kwargs = {} + cacher = SamplesCacher(f'duke_cache_ver{Duke.DUKE_DATASET_VER}', + static_pipeline, + [cache_dir], restart_cache=reset_cache, workers=num_workers, + ignore_nan_inequality=True, + **cache_kwargs) + + my_dataset = DatasetDefault(sample_ids=sample_ids, + static_pipeline=static_pipeline, + dynamic_pipeline=dynamic_pipeline, + cacher=cacher + ) + my_dataset.create() + return my_dataset + + +def remove_entries_with_nan_label(sample_dict, key): + if np.isnan(sample_dict[key]): + print(f"====== {get_sample_id(sample_dict)} has nan label ==> excluded") + return None + return sample_dict + + +def get_selected_sample_ids(): + all_sample_ids = Duke.sample_ids() + excluded_indexes = ['029', '050', '108', '122', '127', '130', '138', '151', '154', '155', '159', '162', '171', '179', '182', '194', + '208', '213', '222', '226', '243', '248', '257', '272', '276', '279', '302', '309', '314', '332', '347', '359', + '367', '382', '388', '391', '406', '422', '434', '447', '449', '470', '524', '549', '553', '555', '571', '579', + '600', '619', '621', '627', '637', '638', '658', '701', '719', '733', '747', '775', '779', '785', '810', '813', + '828', '837', '848', '867', '918', '919'] + excluded_sample_ids = set(['Breast_MRI_' + s for s in excluded_indexes]) + + selected_samples_id = [s for s in all_sample_ids if s not in excluded_sample_ids] + return selected_samples_id diff --git a/fuseimg/datasets/duke_label_type.py b/fuseimg/datasets/duke_label_type.py new file mode 100644 index 000000000..d1382db30 --- /dev/null +++ b/fuseimg/datasets/duke_label_type.py @@ -0,0 +1,92 @@ +from enum import Enum + +import numpy as np +import pandas as pd + + +class DukeLabelType(Enum): + ispCR = 'ispCR' + STAGING_TUMOR_SIZE = 'Staging Tumor Size' + HISTOLOGY_TYPE = 'Histology Type' + IS_HIGH_TUMOR_GRADE_TOTAL = 'is High Tumor Grade Total' + + def get_value(self, clinical_features): + col_name = self.get_column_name() + value = clinical_features[col_name] + process_func = self.get_process_func() + if process_func is None: + return value + if isinstance(value, pd.Series): + value = value.apply(process_func) + else: + value = process_func(value) + return value + + def select_features(self, clinical_features): # todo: ask Tal + group1 = ['MRI Findings:Skin/Nipple Invovlement', # 'Skin Invovlement', + 'US features:Tumor Size (cm)', # 'Tumor Size US', + 'Mammography Characteristics:Tumor Size (cm)', # 'Tumor Size MG', + 'MRI Technical Information:FOV Computed (Field of View) in cm ', # 'Field of View', + 'MRI Technical Information:Contrast Bolus Volume (mL)', # 'Contrast Bolus Volume', + 'Demographics:Race and Ethnicity', # 'Race', + 'MRI Technical Information:Manufacturer Model Name', # 'Manufacturer', + 'MRI Technical Information:Slice Thickness', # 'Slice Thickness'] + ] + if self in (DukeLabelType.ispCR, DukeLabelType.STAGING_TUMOR_SIZE): + fnames = group1 + elif self == DukeLabelType.HISTOLOGY_TYPE: + fname = 'MRI Findings:Multicentric/Multifocal' # 'Multicentric' + fnames = group1 + [fname] + elif self == DukeLabelType.IS_HIGH_TUMOR_GRADE_TOTAL: + fnames = ['Mammography Characteristics:Breast Density', # 'Breast Density MG', + 'Tumor Characteristics:PR', # 'PR', + 'Tumor Characteristics:HER2', # 'HER2', + 'Tumor Characteristics:ER', # 'ER'] + ] + else: + raise NotImplementedError(self) + return clinical_features[fnames] + + def get_column_name(self): + if self == DukeLabelType.ispCR: + return 'Near Complete Response:Overall Near-complete Response: Stricter Definition' # 'Near pCR Strict' + if self == DukeLabelType.STAGING_TUMOR_SIZE: + return 'Tumor Characteristics:Staging(Tumor Size)# [T]' # 'Staging Tumor Size' + if self == DukeLabelType.HISTOLOGY_TYPE: + return 'Tumor Characteristics:Histologic type' # 'Histologic type' + # if self == DukeLabelType.IS_HIGH_TUMOR_GRADE_TOTAL: + # return 'Tumor Grade Total' + raise NotImplementedError(self) + + def get_process_func(self): + if self == DukeLabelType.ispCR: + def update_func(ispCR): + if ispCR > 2: + return np.NaN + if ispCR == 0 or ispCR == 2: + return 0 + else: + return 1 + + return update_func + + if self == DukeLabelType.STAGING_TUMOR_SIZE: + return lambda val: 1 if val > 1 else 0 + + if self == DukeLabelType.HISTOLOGY_TYPE: + def update_func(histology_type): + if histology_type == 1: + return 0 + elif histology_type == 10: + return 1 + else: + return np.NaN + + return update_func + + if self == DukeLabelType.IS_HIGH_TUMOR_GRADE_TOTAL: + return lambda grade: 1 if grade >= 7 else 0 + raise NotImplementedError(self) + + def get_num_classes(self): + return 2 # currrently all are binary classification tasks diff --git a/fuseimg/datasets/prostate_x.py b/fuseimg/datasets/prostate_x.py new file mode 100644 index 000000000..3de04cd32 --- /dev/null +++ b/fuseimg/datasets/prostate_x.py @@ -0,0 +1,510 @@ +import numpy as np +import glob +import os +import logging + +import pickle +from typing import Hashable, Optional, Sequence + +import pandas as pd +from functools import partial + +from fuse.data import DatasetDefault +from fuse.data import PipelineDefault +from fuse.data.datasets.caching.samples_cacher import SamplesCacher +from fuse.data.ops.op_base import OpReversibleBase, OpBase +from fuse.data.ops import ops_read, ops_cast +from fuse.data.utils.sample import get_sample_id +from fuse.utils import NDict +from fuseimg.data.ops import ops_mri +from fuse.data.ops import ops_common +from fuse.data.ops.ops_aug_common import OpSample, OpRandApply +from fuse.utils.rand.param_sampler import RandBool, RandInt, Uniform, Choice + +from fuseimg.data.ops.aug import geometry, geometry3d +from fuse.utils.rand import param_sampler + +from enum import Enum, auto + +import torch + + +def get_selected_series_index(sample_id, seq_id): + patient_id = sample_id[:-2] + map = {'T2': -1, 'ADC': 0, 'ktrans': 0, 'MASK': 0} + if patient_id in ['ProstateX-0148', 'ProstateX-0180']: + map['b'] = [1, 2] + elif patient_id in ['ProstateX-0191']: + map['b'] = [0, 0] + elif patient_id in ['ProstateX-0116']: + map['b'] = [0, 1] + else: + map['b'] = [0, 2] + return map[seq_id] + + +class ProstateXLabelType(Enum): + ClinSig = 'ClinSig' + + def get_features_to_use(self): + if self == ProstateXLabelType.ClinSig: + return ['zone'] + + raise NotImplementedError(self) + + def get_column_name(self): + if self == ProstateXLabelType.ClinSig: + return 'ClinSig' + + def get_process_func(self): + if self == ProstateXLabelType.ClinSig: + return lambda val: 1 if val > 0 else 0 + + raise NotImplementedError(self) + + def get_value(self, clinical_features): + col_name = self.get_column_name() + value = clinical_features[col_name] + 0 # why should be add 0?? + process_func = self.get_process_func() + if process_func is None: + return value + if isinstance(value, pd.Series): + value = value.apply(process_func) + else: + value = process_func(value) + return value + + def get_num_classes(self): + return 2 # currrently all are binary classification tasks + + +class ProstateX: + ProstateX_DATASET_VER = 0 + PATCH_XY_SIZE = 74 + PATCH_Z_SIZE = 13 + + @staticmethod + def sample_ids(data_dir): + annotations_df = get_prostate_x_annotations_df(data_dir) + return annotations_df['Sample ID'].values + + @staticmethod + def static_pipeline(root_path, select_series_func, with_rescale: Optional[bool] = True, + keep_stk_volumes: Optional[bool] = False, verbose: Optional[bool] = True, + annotations_df: Optional[pd.DataFrame] = None) -> PipelineDefault: + + data_path = os.path.join(root_path, 'PROSTATEx') + if annotations_df is None: + annotations_df = get_prostate_x_annotations_df(root_path) + + series_desc_2_sequence_map = get_series_desc_2_sequence_mapping() + dicom_seq_ids = ['T2', 'b', 'b_mix', 'ADC'] + seq_reverse_map = {s: True for s in dicom_seq_ids} + + static_pipeline_steps = [ + # step 1: map sample_ids to MRI folders + (OpProstateXSampleIDDecode(data_path=data_path), + dict(key_path_out='data.input.mri_path', key_patient_id_out='data.input.patient_id')), + # step 2: arrange DICOM files and read info from DICOM tags + (ops_mri.OpExtractDicomsPerSeq(seq_ids=dicom_seq_ids, series_desc_2_sequence_map=series_desc_2_sequence_map, + use_order_indicator=False), + dict(key_in='data.input.mri_path', key_out_seq_ids='data.input.seq_ids', key_out_sequence_prefix='data.input.sequence.')), + # step 3: Load MRI sequences into STK volumes + (ops_mri.OpLoadDicomAsStkVol(seq_reverse_map=seq_reverse_map), + dict(key_in_seq_ids='data.input.seq_ids', key_sequence_prefix='data.input.sequence.')), + # step 4: rename sequence b_mix (if exists) to b; fix certain b sequences + (ops_common.OpLambda(func=partial(ops_mri.rename_seqeunce_from_dict, seq_id_old='b_mix', seq_id_new='b', key_sequence_prefix='data.input.sequence.', + key_seq_ids='data.input.seq_ids')), dict(key=None)), + # step 5: load KTRANS sequences into STK volumes + (ops_mri.OpReadSTKImage(seq_id='ktrans', get_image_file=partial(get_ktrans_image_file_from_sample_id, data_dir=root_path)), + dict(key_sequence_prefix='data.input.sequence.', key_seq_ids='data.input.seq_ids')), + # step 6: select single volumes for b_mix & T2 sequences + (ops_mri.OpSelectVolumes(get_indexes_func=select_series_func, delete_input_volumes=True, selected_seq_ids=['T2', 'b', 'ADC', 'ktrans']), + dict(key_in_seq_ids='data.input.seq_ids', key_in_sequence_prefix='data.input.sequence.', key_out_volumes='data.input.selected_volumes', + key_out_volumes_info='data.input.selected_volumes_info',)), + # step 7: fix certain B volumes + (ops_common.OpLambda(func=partial(fix_certain_b_sequences, key_in_volumes_info='data.input.selected_volumes_info', + key_volumes='data.input.selected_volumes')), dict(key=None)), + # step 8: Register volumes with respect to the first volume + (ops_mri.OpResampleStkVolsBasedRef(reference_inx=0, interpolation='bspline'), dict(key='data.input.selected_volumes')), + # step 9: create a single 4D volume from all the sequences (4th channel is the sequence) + (ops_mri.OpStackList4DStk(delete_input_volumes=True), dict(key_in='data.input.selected_volumes', key_out_volume4d='data.input.volume4D', + key_out_ref_volume='data.input.ref_volume')), + ] + if with_rescale: + # step 10: + static_pipeline_steps += [(ops_mri.OpRescale4DStk(), dict(key='data.input.volume4D'))] + + static_pipeline_steps += [ + # step 11: read tabular data for each patch + (ops_read.OpReadDataframe(data=annotations_df, key_column='Sample ID'), dict(key_out_group='data.input.patch_annotations')), + # step 12: create patch volumes: (i) fixed size around center of annotatins (orig), and (ii) entire annotations + (ops_mri.OpCreatePatchVolumes(lsn_shape=(ProstateX.PATCH_Z_SIZE, ProstateX.PATCH_XY_SIZE, ProstateX.PATCH_XY_SIZE), + name_suffix='_T0', pos_key='pos', lsn_spacing=(3, 0.5, 0.5), + crop_based_annotation=False, delete_input_volumes=not keep_stk_volumes), + dict(key_in_volume4D='data.input.volume4D', + key_in_ref_volume='data.input.ref_volume', + key_in_patch_annotations='data.input.patch_annotations', + key_out='data.input.patch_volume')), + ] + if keep_stk_volumes: + static_pipeline_steps += [ + # step 13: move to ndarray - to allow quick saving + (ops_mri.OpStk2Dict(), + dict(keys=['data.input.patient_id', 'data.input.volume4D', 'data.input.ref_volume'])) + ] + static_pipeline = PipelineDefault("static", static_pipeline_steps, verbose=verbose) + + return static_pipeline + + @staticmethod + def dynamic_pipeline(train: bool, label_type: Optional[ProstateXLabelType] = None, num_channels: int = 5, verbose: Optional[bool] = True): + volume_key = 'data.input.patch_volume' + dynamic_steps = [(ops_cast.OpToTensor(), dict(key=volume_key, dtype=torch.float32))] + + keys_2_keep = [volume_key, 'data.input.patient_id'] + if label_type is not None: + key_ground_truth = 'data.ground_truth' + dynamic_steps.append((OpAdProstateXLabelAndClinicalFeatures(label_type=label_type), + dict(key_in='data.input.patch_annotations', key_out_gt=key_ground_truth, + key_out_clinical_features='data.clinical_features'))) + keys_2_keep.append(key_ground_truth) + + dynamic_steps.append((ops_common.OpKeepKeypaths(), dict(keep_keypaths=keys_2_keep))) + + # augmentation + + if train: + dynamic_steps += [ + # [ + # ('data.input',), + # rotation_in_3d, + # {'z_rot': Uniform(-5.0, 5.0), 'y_rot': Uniform(-5.0, 5.0), 'x_rot': Uniform(-5.0, 5.0)}, + # {'apply': RandBool(0.5)} + # ], + (OpRandApply(OpSample(geometry3d.OpRotation3D()), 0.5), + dict(key='data.input.patch_volume', + ax1_rot=Uniform(-5.0, 5.0), + ax2_rot=Uniform(-5.0, 5.0), + ax3_rot=Uniform(-5.0, 5.0))), + + # [ + # ('data.input',), + # squeeze_3d_to_2d, + # {'axis_squeeze': 'z'}, + # {} + # ], + (geometry.OpAugSqueeze3Dto2D(), dict(key='data.input.patch_volume', axis_squeeze=1)), + + # [ + # ('data.input',), + # aug_op_affine, + # {'rotate': Uniform(0, 360.0), + # 'translate': (RandInt(-4, 4), RandInt(-4, 4)), + # 'flip': (RandBool(0.5), RandBool(0.5)), + # 'scale': Uniform(0.9, 1.1), + # }, + # {'apply': RandBool(0.5)} + # ], + (OpRandApply(OpSample(geometry.OpAugAffine2D()), 0.5), dict( + key="data.input.patch_volume", + rotate=Uniform(0, 360.0), + scale=Uniform(0.9, 1.1), + flip=(RandBool(0.5), RandBool(0.5)), + translate=(RandInt(-4, 4), RandInt(-4, 4)) + )), + + # [ + # ('data.input',), + # aug_op_affine, + # {'rotate': Uniform(-3.0, 3.0), + # 'translate': (RandInt(-2, 2), RandInt(-2, 2)), + # 'flip': (False, False), + # 'scale': Uniform(0.9, 1.1), + # 'channels': Choice(image_channels, probabilities=None)}, + # {'apply': RandBool(0.5) if train_common_params['data.aug.phase_misalignment'] else 0} + # ], + (OpRandApply(OpSample(geometry.OpAugAffine2D()), 0.5), dict( + key="data.input.patch_volume", + rotate=Uniform(-3.0, 3.0), + scale=Uniform(0.9, 1.1), + flip=(False, False), + translate=(RandInt(-2, 2), RandInt(-2, 2)), + channels=Choice([list(range(0, ProstateX.PATCH_Z_SIZE))]) # todo: but we are 2D - there are no channels?? + )), + + # [ + # ('data.input',), + # unsqueeze_2d_to_3d, + # {'channels': num_channels, 'axis_squeeze': 'z'}, + # {} + # ], + + (geometry.OpAugUnsqueeze3DFrom2D(), dict(key='data.input.patch_volume', + axis_squeeze=1, channels=num_channels, + )), + + ] + + dynamic_pipeline = PipelineDefault("dynamic", dynamic_steps, verbose=verbose) + + return dynamic_pipeline + + @staticmethod + def dataset(label_type: Optional[ProstateXLabelType] = None, train: Optional[bool] = False, cache_dir: Optional[str] = None, + data_dir: Optional[str] = None, + select_series_func=get_selected_series_index, num_channels: int = 5, reset_cache: bool = False, num_workers: int = 10, + sample_ids: Optional[Sequence[Hashable]] = None, verbose: Optional[bool] = True, + annotations_df: Optional[pd.DataFrame] = None, + cache_kwargs: Optional[dict] = None) -> DatasetDefault: + + """ + :param label_type: type of label to use + :param cache_dir: path to store the cache of the static pipeline + :param data_dir: path to the original data + :param select_series_func: which series to select for DCE_mix sequences + :param reset_cache: + :param num_workers: number of processes used for caching + :param sample_ids: list of selected patient_ids for the dataset + :return: + """ + + if data_dir is None: + data_dir = "/projects/msieve/MedicalSieve/PatientData/ProstateX/manifest-A3Y4AE4o5818678569166032044/" + + if sample_ids is None: + sample_ids = ProstateX.sample_ids(data_dir) + + static_pipeline = ProstateX.static_pipeline(root_path=data_dir, select_series_func=select_series_func, + annotations_df=annotations_df, verbose=verbose) + dynamic_pipeline = ProstateX.dynamic_pipeline(train=train, label_type=label_type, num_channels=num_channels, verbose=verbose) + + if cache_dir is None: + cacher = None + else: + if cache_kwargs is None: + cache_kwargs = {} + cacher = SamplesCacher(f'prostate_x_cache_ver{ProstateX.ProstateX_DATASET_VER}', + static_pipeline, + [cache_dir], restart_cache=reset_cache, workers=num_workers, + ignore_nan_inequality=True, + **cache_kwargs) + + my_dataset = DatasetDefault(sample_ids=sample_ids, + static_pipeline=static_pipeline, + dynamic_pipeline=dynamic_pipeline, + cacher=cacher + ) + my_dataset.create() + return my_dataset + + +def fix_certain_b_sequences(sample_dict, key_in_volumes_info, key_volumes): + volumes_info = sample_dict[key_in_volumes_info] + volumes = sample_dict[key_volumes] + + B_SER_FIX = ['diffusie-3Scan-4bval_fs', + 'ep2d_DIFF_tra_b50_500_800_1400_alle_spoelen', + 'diff tra b 50 500 800 WIP511b alle spoelen'] + # seq = ['T2', 'b400', 'b800', 'ADC', 'ktrans'] + # if ('b' in seq_info.keys()): + # if (seq_info['b'][0] in B_SER_FIX): + # vols_list[seq.index('b800')].CopyInformation(vols_list[seq.index('ADC')]) + # vols_list[seq.index('b400')].CopyInformation(vols_list[seq.index('ADC')]) + + has_b_to_fix = np.any([(info['seq_id'] == 'b') and (info['series_desc'] in B_SER_FIX) for info in volumes_info]) + if has_b_to_fix: + for i, val in zip([1, 2], [400, 800]): + volumes[i].CopyInformation(volumes[3]) + assert volumes_info[i]['seq_id'] == 'b' + assert volumes_info[i]['series_desc'] in B_SER_FIX + val2 = volumes_info[i]['dicoms_id'] + if val == val2: + print(f"fix B OK 8888888888 : {val}={val2}") + else: + print(f"fix B mismatch 8888888888 : {val}!={val2}") + + return sample_dict + + +def get_ktrans_image_file_from_sample_id(sample_id, data_dir): + patient_id = sample_id.split('_')[0] + ktrans_patient_dir = os.path.join(data_dir, 'ProstateXKtrains-train-fixed', patient_id) + ktrans_mhd_files = glob.glob(os.path.join(ktrans_patient_dir, '*.mhd')) + assert len(ktrans_mhd_files) == 1 + return ktrans_mhd_files[0] + + +class OpProstateXSampleIDDecode(OpReversibleBase): + ''' + decodes sample id into path of MRI images + ''' + + def __init__(self, data_path: str, **kwargs): + super().__init__(**kwargs) + self._data_path = data_path + + def __call__(self, sample_dict: NDict, key_path_out: str, key_patient_id_out:str, op_id: Optional[str]) -> NDict: + sid = get_sample_id(sample_dict) + + patient_id = sid[:-2] + sample_dict[key_patient_id_out] = get_sample_path(self._data_path, patient_id) + sample_dict[key_path_out] = get_sample_path(self._data_path, patient_id) + + + return sample_dict + + def reverse(self, sample_dict: dict, key_to_reverse: str, key_to_follow: str, op_id: Optional[str]) -> dict: + return sample_dict + + +class OpAdProstateXLabelAndClinicalFeatures(OpBase): + ''' + decodes sample id into path of MRI images + ''' + + def __init__(self, label_type: ProstateXLabelType, is_concat_features_to_input: Optional[bool] = False, **kwargs): + super().__init__(**kwargs) + self._label_type = label_type + self._is_concat_features_to_input = is_concat_features_to_input + + def __call__(self, sample_dict: NDict, key_in: str, + key_out_gt: str, key_out_clinical_features: str) -> NDict: + clinical_features = sample_dict[key_in] + label_val = self._label_type.get_value(clinical_features) + if np.isnan(label_val): + return None # should filter example (instead of sample_dict['data.filter'] = True ) + + label_tensor = torch.tensor(label_val, dtype=torch.int64) + sample_dict[key_out_gt] = label_tensor # 'data.ground_truth' + + if False: + # add clinical + features_to_use = self._label_type.get_features_to_use() + + zone2feature = { + 'PZ': torch.tensor(np.array([0, 0, 0]), dtype=torch.float32), + 'TZ': torch.tensor(np.array([0, 0, 1]), dtype=torch.float32), + 'AS': torch.tensor(np.array([0, 1, 0]), dtype=torch.float32), + 'SV': torch.tensor(np.array([1, 0, 0]), dtype=torch.float32), + } + + clinical_features_to_use = zone2feature[clinical_features[features_to_use[0]]] + + sample_dict[key_out_clinical_features] = clinical_features_to_use # 'data.clinical_features' + + if self._is_concat_features_to_input: + # select input channel + input_tensor = sample_dict['data.input'] + input_shape = input_tensor.shape + for feature in clinical_features_to_use: + input_tensor = torch.cat( + (input_tensor, feature.repeat(input_shape[1], input_shape[2], input_shape[3]).unsqueeze(0)), dim=0) + sample_dict['data.input'] = input_tensor + + return sample_dict + + +def get_prostate_x_annotations_df(data_dir): + if True: + # v5 + annotations_df = pd.read_csv(os.path.join(data_dir, 'Lesion Information', 'ProstateX-Findings-Train.csv')) + annotations_df['Patient ID'] = annotations_df['ProxID'] + annotations_df = annotations_df.set_index('ProxID') + annotations_df = annotations_df[[ 'Patient ID', 'fid','ClinSig', 'pos', 'zone']] + # ['ProstateX-0005_1', + # 'ProstateX-0025_1', 'ProstateX-0025_2', 'ProstateX-0025_3', 'ProstateX-0025_4', + # 'ProstateX-0105_2', 'ProstateX-0105_3', 'ProstateX-0154_3'] + + vals = [('ProstateX-0005', 1), # there are two of this + ('ProstateX-0105', 2), ('ProstateX-0105', 3), + ('ProstateX-0154', 3)] + a_filter = np.zeros(annotations_df.shape[0], dtype=bool) + for pid, fid in vals: + a_filter |=( annotations_df['Patient ID'] == pid) & (annotations_df['fid']==fid) + assert a_filter.sum() == 5 + annotations_df = annotations_df[~a_filter] + + else: + #v6 + PROSTATEX_PROCESSED_FILE_DIR = '/projects/msieve_dev3/usr/Tal/prostate_x_processed_files' + + annotations_path = os.path.join(PROSTATEX_PROCESSED_FILE_DIR, 'dataset_prostate_x_folds_ver29062021_seed1.pickle') + with open(annotations_path, 'rb') as infile: + fold_annotations_dict = pickle.load(infile) + annotations_df = pd.concat( + [fold_annotations_dict[f'data_fold{fold}'] for fold in range(len(fold_annotations_dict))]) + annotations_df = annotations_df[['Patient ID', 'fid', 'ClinSig', 'pos', 'zone']] + + pids_to_fix = [('ProstateX-0159', 3)] + + + # fix a bug in the dataset of wrong indexing + for pid, n_fid in pids_to_fix: + a_filter = annotations_df['Patient ID'] == pid + assert a_filter.sum() == n_fid + annotations_df.loc[a_filter, 'fid'] = np.arange(1, n_fid+1) + annotations_df['Sample ID'] = annotations_df['Patient ID']+ '_'+annotations_df['fid'].astype(str) + + # filter problematic samples: + problematic_patient_ids = ['ProstateX-0025'] + a_filter = ~annotations_df[ 'Patient ID'].isin(problematic_patient_ids) + if not np.all(a_filter): + annotations_df = annotations_df[a_filter] + return annotations_df + + +def get_samples_for_debug(n_pos, n_neg, label_type): + annotations_df = get_prostate_x_annotations_df() + label_values = label_type.get_value(annotations_df) + patient_ids = annotations_df['Sample ID'] + sample_ids = [] + for label_val, n_vals in zip([True, False], [n_pos, n_neg]): + sample_ids += patient_ids[label_values == label_val].values.tolist()[:n_vals] + return sample_ids + + +def get_series_desc_2_sequence_mapping(): + series_desc_2_sequence_mapping = \ + { + 't2_tse_tra': 'T2', + 't2_tse_tra_Grappa3': 'T2', + 't2_tse_tra_320_p2': 'T2', + + 'ep2d-advdiff-3Scan-high bvalue 100': 'b', + 'ep2d-advdiff-3Scan-high bvalue 500': 'b', + 'ep2d-advdiff-3Scan-high bvalue 1400': 'b', + 'ep2d_diff_tra2x2_Noise0_FS_DYNDISTCALC_BVAL': 'b', + + 'ep2d_diff_tra_DYNDIST': 'b_mix', + 'ep2d_diff_tra_DYNDIST_MIX': 'b_mix', + 'diffusie-3Scan-4bval_fs': 'b_mix', + 'ep2d_DIFF_tra_b50_500_800_1400_alle_spoelen': 'b_mix', + 'diff tra b 50 500 800 WIP511b alle spoelen': 'b_mix', + + 'ep2d_diff_tra_DYNDIST_MIX_ADC': 'ADC', + 'diffusie-3Scan-4bval_fs_ADC': 'ADC', + 'ep2d-advdiff-MDDW-12dir_spair_511b_ADC': 'ADC', + 'ep2d-advdiff-3Scan-4bval_spair_511b_ADC': 'ADC', + 'ep2d_DIFF_tra_b50_500_800_1400_alle_spoelen_ADC': 'ADC', + 'diff tra b 50 500 800 WIP511b alle spoelen_ADC': 'ADC', + 'ADC_S3_1': 'ADC', + 'ep2d_diff_tra_DYNDIST_ADC': 'ADC', + + } + + return series_desc_2_sequence_mapping + + +def get_sample_path(data_path, patient_id): + patient_path_pattern = os.path.join(data_path, patient_id, '*') + patient_path = sorted(glob.glob(patient_path_pattern)) + if len(patient_path) > 1: + lgr = logging.getLogger('Fuse') + lgr.warning( f'{patient_id} has {len(patient_path)} files. Taking first') + return patient_path[0] + + + +################################## + diff --git a/requirements.txt b/requirements.txt index 7aa10304e..3b11a6725 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,3 +33,5 @@ medpy pytorch_lightning hydra-core omegaconf +pyradiomics +openpyxl \ No newline at end of file