Greater parralellization of dirty_map

core/algebra.py

@@ -563,6 +563,84 @@ def load_h5(h5obj, path):
     iarray = info_array(iarray, info)
     return iarray
 
+def load_h5_memmap(h5obj, path, fname, gb_lim):
+    """Load a info memmap from an hdf5 file.
+
+    Parameters
+    ----------
+    h5obj : h5py File or Group object
+    File from which the info array will be read from.
+    path : string
+    Path within `h5obj` to read the array.
+
+    Returns
+    -------
+    memarray : info_memmap
+    Array loaded from file.
+    """
+
+    # TODO: Allow `h5obj` to be a string with a path to a file to be opened
+    # and then closed.
+    #data = h5obj[path]
+    dset = h5obj[path]
+    memarray = sp.memmap(fname, shape=dset.shape, dtype=dset.dtype, mode='w+')
+    memarray = fill_memmap( memarray, 1000000000*gb_lim, dset)
+    #memarray = data.value
+    info = {}
+    for key, value in dset.attrs.iteritems():
+        info[key] = safe_eval(value)
+    memarray = info_memmap(memarray, info)
+    return memarray
+
+def fill_memmap(memmap, mem_lim, dset):
+    chunk_size = dset.size*dset.dtype.itemsize
+    for ind in range(len(dset.shape)):
+        if chunk_size <= mem_lim:
+            i = ind
+            print str(i)
+            break
+        else:
+            chunk_size = chunk_size/dset.shape[ind]
+    if i == 0:
+        memmap = dset.value
+    else:
+        index_list = []
+        for num in range(i):
+            index_list.append(range(dset.shape[num]))
+        indexes = cross(index_list)
+        for index in indexes:
+            print index
+            memmap[tuple(index)] = dset[tuple(index)]
+    return memmap
+
+
+def cross(set_list):
+        '''Given a list of sets, return the cross product.'''
+        # By associativity of cross product, cross the first two sets together
+        # then cross that with the rest. The big conditional in the list
+        # comprehension is just to make sure that there are no nested lists
+        # in the final answer.
+        if len(set_list) == 1:
+                ans = []
+                for elem in set_list[0]:
+                        # In the 1D case, these elements are not lists.
+                        if type(elem) == list:
+                                ans.append(np.array(elem))
+                        else:
+                                ans.append(np.array([elem]))
+                return ans
+        else:
+                A = set_list[0]
+                B = set_list[1]
+                cross_2 = [a+b if ((type(a) == list) and (type(b) == list)) else \
+                (a+[b] if type(a) == list else ([a]+b if type(b) == list else \
+                [a]+[b])) for a in A for b in B]
+                remaining = set_list[2:]
+                remaining.insert(0,cross_2)
+                return cross(remaining)
+
+
+
 
 # ---- Functions for manipulating above arrays as matrices and vectors. -------
 

input/km/cm_parkes_parallel2.ini

@@ -0,0 +1,103 @@
+# Input file for map maker testing.
+
+import os
+import glob
+
+#from core import dir_data
+
+
+#sessions = range(41, 81)
+#file_list = open('scripts/datalist_2008_center.txt', 'r').read().splitlines() 
+#file_list = file_list[:4]
+file_middles = []
+#for file in file_list:
+#    file_middles.append(file[72:-7])
+
+data_dir = '/scratch/p/pen/andersoc/second_parkes_pipe/rebinned/'
+dir_len = len(data_dir)
+for el in os.walk(data_dir):
+    dir_files = glob.glob(el[0] + '/*' + '2df1' + '*.fits')
+    for file in dir_files:
+        file_middles.append(file.split('.')[0][dir_len:])
+
+
+map_centre = (10, -30)
+# Large problem size, full map.
+#map_shape = (90, 48)
+#map_spacing = .0627
+# Standard problem size.
+#map_shape = (72, 38)
+#map_spacing = .0627
+# Small problem size.
+#map_shape = (44, 24)
+#map_spacing = .0627
+# Tiny problem size.
+map_shape = (188, 88)
+map_spacing = .08
+
+# Data paths
+#raid_pro = os.getenv("RAID_PRO")
+input_data_dir = data_dir
+#base_dir = '/scratch/p/pen/andersoc/first_parkes_pipe/' 
+base_dir = '/scratch/p/pen/andersoc/second_parkes_pipe/'
+map_dir = base_dir + 'maps/'
+#map_dir = base_dir + 'parallel_maps/'
+#map_root = map_dir + "parkes_test_parallel_thread_ra30decn30_small_"
+map_root = map_dir + "parkes_parallel_thread_ra20decn30_p08_125by88_beam1_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam2_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam3_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam4_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam5_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam6_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam7_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam8_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam9_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam10_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam11_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam12_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam13_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_312by88_beam2_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_312by88_beam3_"
+
+# Map maker parameters.
+#dm_input_root = '/scratch/p/pen/andersoc/first_parkes_pipe/rotated_to_I/'
+dm_input_root = input_data_dir
+dm_file_middles = file_middles
+dm_input_end = '.fits'
+dm_output_root = map_root
+dm_scans = ()
+dm_IFs = (0,)
+#dm_beams = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
+#dm_beam = 1
+#dm_beam = 2
+dm_beam = 3
+
+dm_thread_divide = True
+dm_polarizations = ('XX', 'YY')
+#dm_polarizations = ('XX',)
+dm_field_centre = map_centre
+dm_pixel_spacing = map_spacing
+dm_map_shape = map_shape
+dm_time_block = 'scan'
+dm_n_files_group = 10
+#dm_frequency_correlations = 'measured'
+dm_frequency_correlations = 'None'
+dm_number_frequency_modes = 0
+#dm_number_frequency_modes_discard = 1
+#dm_noise_parmeters_input_root = 'None'
+dm_noise_parameter_file = ''
+dm_deweight_time_mean = True
+dm_deweight_time_slope = True
+dm_interpolation = 'linear'
+#dm_ts_foreground_mode_file = ''
+#dm_n_ts_foreground_modes = 0
+
+cm_input_root = dm_output_root
+cm_output_root = cm_input_root
+cm_polarizations = ('XX','YY')
+#cm_polarizations = ('XX',)
+#cm_polarizations = ('YY',)
+#cm_polarizations = ('I',)
+cm_save_noise_diag = True
+cm_bands = (1316,)
+#cm_bands = (1315,)

input/km/dm_parkes_thread2.ini

@@ -0,0 +1,113 @@
+# Input file for map maker testing.
+
+import os
+import glob
+
+#from core import dir_data
+
+
+#sessions = range(41, 81)
+#file_list = open('scripts/datalist_2008_center.txt', 'r').read().splitlines() 
+#file_list = file_list[:4]
+file_middles = []
+#for file in file_list:
+#    file_middles.append(file[72:-7])
+
+data_dir = '/scratch/p/pen/andersoc/second_parkes_pipe/rebinned/'
+dir_len = len(data_dir)
+for el in os.walk(data_dir):
+    dir_files = glob.glob(el[0] + '/*' + '2df1' + '*.fits')
+    for file in dir_files:
+        file_middles.append(file.split('.')[0][dir_len:])
+
+
+#map_centre = (10, -30)
+#map_centre = (20, -30)
+map_centre = (-10, -30)
+# Large problem size, full map.
+#map_shape = (90, 48)
+#map_spacing = .0627
+# Standard problem size.
+#map_shape = (72, 38)
+#map_spacing = .0627
+# Small problem size.
+#map_shape = (44, 24)
+#map_spacing = .0627
+# Tiny problem size.
+#map_shape = (375, 100)
+map_shape = (125, 88)
+map_spacing = .08
+
+# Data paths
+#raid_pro = os.getenv("RAID_PRO")
+input_data_dir = data_dir
+#base_dir = '/scratch/p/pen/andersoc/first_parkes_pipe/' 
+base_dir = '/scratch/p/pen/andersoc/second_parkes_pipe/'
+map_dir = base_dir + 'maps/'
+#map_dir = base_dir + 'parallel_maps/'
+#map_root = map_dir + "parkes_test_parallel_thread_ra30decn30_small_"
+#map_root = map_dir + "parkes_parallel_thread_ra20decn30_p08_375by100_beam1_"
+#map_root = map_dir + "parkes_parallel_thread_ra20decn30_p08_125by88_beam2_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam3_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam4_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam5_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam6_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam7_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam8_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam9_"
+map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam10_testsave_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam11_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam12_"
+#map_root = map_dir + "parkes_parallel_thread_ra10decn30_p08_125by88_beam13_"
+
+# Map maker parameters.
+#dm_input_root = '/scratch/p/pen/andersoc/first_parkes_pipe/rotated_to_I/'
+dm_input_root = input_data_dir
+dm_file_middles = file_middles
+dm_input_end = '.fits'
+dm_output_root = map_root
+dm_scans = ()
+dm_IFs = (0,)
+#dm_beams = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
+#dm_beam = 1
+#dm_beam = 2
+#dm_beam = 3
+#dm_beam = 4
+#dm_beam = 5
+#dm_beam = 6
+#dm_beam = 7
+#dm_beam = 8
+#dm_beam = 9
+dm_beam = 10
+#dm_beam = 11
+#dm_beam = 12
+#dm_beam = 13
+
+dm_thread_divide = True
+#dm_polarizations = ('XX', 'YY')
+dm_polarizations = ('XX',)
+#dm_polarizations = ('YY',)
+dm_field_centre = map_centre
+dm_pixel_spacing = map_spacing
+dm_map_shape = map_shape
+dm_time_block = 'scan'
+dm_n_files_group = 10
+#dm_frequency_correlations = 'measured'
+dm_frequency_correlations = 'None'
+dm_number_frequency_modes = 0
+#dm_number_frequency_modes_discard = 1
+#dm_noise_parmeters_input_root = 'None'
+dm_noise_parameter_file = ''
+dm_deweight_time_mean = True
+dm_deweight_time_slope = True
+dm_interpolation = 'linear'
+#dm_ts_foreground_mode_file = ''
+#dm_n_ts_foreground_modes = 0
+
+cm_input_root = dm_output_root
+cm_output_root = cm_input_root
+cm_polarizations = ('XX','YY')
+#cm_polarizations = ('I',)
+cm_save_noise_diag = True
+cm_bands = (1316,)
+#cm_bands = (1315,)

input/km/dm_parkes_thread_test.ini

@@ -0,0 +1,64 @@
+# Input file for map maker testing.
+
+import os
+
+from core import dir_data
+
+
+sessions = range(41, 81)
+file_list = open('scripts/datalist_2008_center.txt', 'r').read().splitlines() 
+#file_list = file_list[:4]
+file_middles = []
+for file in file_list:
+    file_middles.append(file[72:-7])
+
+
+map_centre = (30, -30)
+# Large problem size, full map.
+#map_shape = (90, 48)
+#map_spacing = .0627
+# Standard problem size.
+#map_shape = (72, 38)
+#map_spacing = .0627
+# Small problem size.
+#map_shape = (44, 24)
+#map_spacing = .0627
+# Tiny problem size.
+map_shape = (10, 10)
+map_spacing = 1
+
+# Data paths
+#raid_pro = os.getenv("RAID_PRO")
+input_data_dir = '/scratch/p/pen/andersoc/first_parkes_pipe/rotated_to_I/' 
+base_dir = '/scratch/p/pen/andersoc/first_parkes_pipe/'
+map_dir = base_dir + 'parallel_maps/'
+map_root = map_dir + "parkes_test_parallel_thread_ra30decn30_small_"
+
+
+# Map maker parameters.
+dm_input_root = '/scratch/p/pen/andersoc/first_parkes_pipe/rotated_to_I/'
+dm_file_middles = file_middles
+dm_input_end = '.fits'
+dm_output_root = map_root
+dm_scans = ()
+dm_IFs = (0,)
+dm_beams = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
+
+dm_thread_divide = True
+dm_polarizations = ('I',)
+dm_field_centre = map_centre
+dm_pixel_spacing = map_spacing
+dm_map_shape = map_shape
+dm_time_block = 'scan'
+dm_n_files_group = 10
+#dm_frequency_correlations = 'measured'
+dm_frequency_correlations = 'None'
+dm_number_frequency_modes = 0
+#dm_number_frequency_modes_discard = 1
+#dm_noise_parmeters_input_root = 'None'
+dm_noise_parameter_file = ''
+dm_deweight_time_mean = True
+dm_deweight_time_slope = True
+dm_interpolation = 'linear'
+#dm_ts_foreground_mode_file = ''
+#dm_n_ts_foreground_modes = 0

input/km/mm_test.ini

@@ -32,11 +32,13 @@ raid_pro = os.getenv("RAID_PRO")
 input_data_dir = raid_pro + "kiyo/gbt_out_new/"
 base_dir = os.getenv("GBT_OUT")
 map_dir = base_dir + 'maps/first_lockandwrite_test/'
+#map_dir = base_dir + 'maps/first_lockandwrite_test/tcs_threading/'
 #map_root = map_dir + "tmp_test_parallel_freqcorr_"
 map_root = map_dir + "tmp_test_parallel_inherit_"
 #map_root = map_dir + "tmp_test_regular_"
 
 # Map maker parameters.
+dm_thread_divide = False
 dm_input_root = input_data_dir + 'reflagged_sec/'
 dm_file_middles = file_middles
 dm_input_end = '.fits'
@@ -48,7 +50,7 @@ dm_field_centre = map_centre
 dm_pixel_spacing = map_spacing
 dm_map_shape = map_shape
 dm_time_block = 'scan'
-dm_n_files_group = 280  # tpb nodes.
+dm_n_files_group = 10
 #dm_frequency_correlations = 'measured'
 dm_frequency_correlations = 'None'
 dm_number_frequency_modes = 3