From 307523e941ee5d7c131a18bd56bdc55bf04d4228 Mon Sep 17 00:00:00 2001 From: Micah Bowles Date: Wed, 31 Mar 2021 12:15:10 +0100 Subject: [PATCH 01/39] Cluster addition: Galahad Added `--cluster` argument (ilifu or galahad, but could be expanded to run as a standard on other slurm based HPC). --- processMeerKAT/processMeerKAT.py | 107 +++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index bb332b6..c78e6a5 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -171,6 +171,7 @@ def parse_scripts(val): parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) + parser.add_argument("--cluster",metavar='name', required=False, type=str, default="ilifu", help="Name of cluster being used [default: ilifu; allowed: galahad, ilifu]") parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") parser.add_argument("-C","--config",metavar="path", default=CONFIG, required=False, type=str, help="Relative (not absolute) path to config file.") parser.add_argument("-N","--nodes",metavar="num", required=False, type=int, default=1, @@ -266,6 +267,10 @@ def validate_args(args,config,parser=None): parser : class ``argparse.ArgumentParser``, optional If this is input, parser error will be raised.""" + if args['cluster'] not in ['ilifu','galahad']: + msg = "The selected cluster must be one of [ilifu, galahad]. Pipeline has not been implemented for other clusters yet." + raise_error(config, msg, parser) + if parser is None or args['build']: if args['MS'] is None and not args['nofields']: msg = "You must input an MS [-M --MS] to build the config file." @@ -396,7 +401,6 @@ def write_command(script,args,name='job',mpi_wrapper=MPI_WRAPPER,container=CONTA def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="job",runname='',plane=1,exclude='',mpi_wrapper=MPI_WRAPPER, container=CONTAINER,partition="Main",time="12:00:00",casa_script=True,casacore=False,SPWs='',nspw=1,account='b03-idia-ag',reservation=''): - """Write a SLURM sbatch file calling a certain script (and args) with a particular configuration. Arguments: @@ -495,23 +499,38 @@ def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="jo if 'selfcal' in script: params['command'] = 'ulimit -n 16384\n' + params['command'] - contents = """#!/bin/bash{array}{exclude}{reservation} - #SBATCH --account={account} - #SBATCH --nodes={nodes} - #SBATCH --ntasks-per-node={tasks} - #SBATCH --cpus-per-task={cpus} - #SBATCH --mem={mem}GB - #SBATCH --job-name={runname}{name} - #SBATCH --distribution=plane={plane} - #SBATCH --output={LOG_DIR}/%x-{ID}.out - #SBATCH --error={LOG_DIR}/%x-{ID}.err - #SBATCH --partition={partition} - #SBATCH --time={time} - - export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK - - {command}""" - + if params['cluster']=='ilifu': + contents = """#!/bin/bash{array}{exclude}{reservation} + #SBATCH --account={account} + #SBATCH --nodes={nodes} + #SBATCH --ntasks-per-node={tasks} + #SBATCH --cpus-per-task={cpus} + #SBATCH --mem={mem}GB + #SBATCH --job-name={runname}{name} + #SBATCH --distribution=plane={plane} + #SBATCH --output={LOG_DIR}/%x-{ID}.out + #SBATCH --error={LOG_DIR}/%x-{ID}.err + #SBATCH --partition={partition} + #SBATCH --time={time} + + export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK + + {command}""" + elif params['cluster'] == 'galahad': + contents="""#!/bin/bash{array}{exclude}{reservation} + #SBATCH --nodes={nodes} + #SBATCH --threads=16 + #SBATCH --cpus-per-task=1 + #SBATCH --mem=1000GB + #SBATCH --job-name={runname}{name} + #SBATCH --output={LOG_DIR}/%x-{ID}.out + #SBATCH --error={LOG_DIR}/%x-{ID}.err + #SBATCH --partition={partition} + #SBATCH --time={time} + #SBATCH -w compute-0-100 + + {command} + """ #insert arguments and remove whitespace contents = contents.format(**params).replace(" ","") @@ -521,6 +540,9 @@ def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="jo config.write(contents) config.close() + if params['cluster'] == 'galahad': + print("Galahad sbatch file content formated as:\n{0}".format(contents)) + logger.debug('Wrote sbatch file "{0}"'.format(sbatch)) def write_spw_master(filename,config,SPWs,precal_scripts,postcal_scripts,submit,dir='jobScripts',pad_length=5,dependencies='',timestamp='',slurm_kwargs={}): @@ -1435,6 +1457,55 @@ def main(): args = parse_args() setup_logger(args.config,args.verbose) + # Cluster adaptations if required + if args.cluser=='galahad': + print('Configuring pipeline for use on Galahad ...') + args['partition']='WHEEL' + args['mem']=1000 + MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1000 + MEM_PER_NODE_GB_LIMIT = 1000 + # Set global limits for current ilifu cluster configuration + TOTAL_NODES_LIMIT = 1 + CPUS_PER_NODE_LIMIT = 16 + NTASKS_PER_NODE_LIMIT = CPUS_PER_NODE_LIMIT + MEM_PER_NODE_GB_LIMIT = 1000 #237568 MB + MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1300 #491520 MB + + # Set global values for paths and file names + THIS_PROG = __file__ + SCRIPT_DIR = os.path.dirname(THIS_PROG) + LOG_DIR = 'logs' + CALIB_SCRIPTS_DIR = 'crosscal_scripts' + AUX_SCRIPTS_DIR = 'aux_scripts' + SELFCAL_SCRIPTS_DIR = 'selfcal_scripts' + CONFIG = 'default_config.txt' + TMP_CONFIG = '.config.tmp' + MASTER_SCRIPT = 'submit_pipeline.sh' + + #Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values + FIELDS_CONFIG_KEYS = ['fluxfield','bpassfield','phasecalfield','targetfields','extrafields'] + CROSSCAL_CONFIG_KEYS = ['minbaselines','chanbin','width','timeavg','createmms','keepmms','spw','nspw','calcrefant','refant','standard','badants','badfreqranges'] + SELFCAL_CONFIG_KEYS = ['nloops','restart_no','cell','robust','imsize','wprojplanes','niter','threshold','multiscale','nterms','gridder','deconvolver','solint','calmode','atrous'] + SLURM_CONFIG_STR_KEYS = ['container','mpi_wrapper','partition','time','name','dependencies','exclude','account','reservation'] + SLURM_CONFIG_KEYS = ['nodes','ntasks_per_node','mem','plane','submit','precal_scripts','postcal_scripts','scripts','verbose'] + SLURM_CONFIG_STR_KEYS + CONTAINER = '/share/nas/mbowles/mightee/casa-stable.simg' + MPI_WRAPPER = CONTAINER # Fairly certain this shouldnt work, but it might. + PRECAL_SCRIPTS = [('calc_refant.py',False,''),('partition.py',True,'')] #Scripts run before calibration at top level directory when nspw > 1 + POSTCAL_SCRIPTS = [('concat.py',False,''),('plotcal_spw.py', False, ''),('selfcal_part1.py',True,''),('selfcal_part2.py',False,''),('run_bdsf.py', False, ''),('make_pixmask.py', False, '')] #Scripts run after calibration at top level directory when nspw > 1 + SCRIPTS = [ ('validate_input.py',False,''), + ('flag_round_1.py',True,''), + ('calc_refant.py',False,''), + ('setjy.py',True,''), + ('xx_yy_solve.py',False,''), + ('xx_yy_apply.py',True,''), + ('flag_round_2.py',True,''), + ('xx_yy_solve.py',False,''), + ('xx_yy_apply.py',True,''), + ('split.py',True,''), + ('quick_tclean.py',True,''), + ('plot_solutions.py',False,'')] + + #Mutually exclusive arguments - display version, build config file or run pipeline if args.version: logger.info('This is version {0}'.format(__version__)) From 70b5272f5551283ec5c0e0904beaa9dbc1af28bb Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 4 Oct 2021 14:50:12 +0100 Subject: [PATCH 02/39] Ignoring venv --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5965412..b1d66dc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build/ dist/ +venv/ From f66adf3dde68f39612f3dbbeaf57c8a5135182a6 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 4 Oct 2021 14:51:25 +0100 Subject: [PATCH 03/39] Move HPC params to DEFAULTS Read in default / limiting values from a default config which specifies which HPC facility has which limitations in their nodes etc. This should eventually allow for easy addition of further slurm based HPCs. --- processMeerKAT/DEFAULTS.cfg | 145 +++++++++++++++++++ processMeerKAT/processMeerKAT.py | 230 +++++++++++++------------------ 2 files changed, 239 insertions(+), 136 deletions(-) create mode 100644 processMeerKAT/DEFAULTS.cfg diff --git a/processMeerKAT/DEFAULTS.cfg b/processMeerKAT/DEFAULTS.cfg new file mode 100644 index 0000000..d87923a --- /dev/null +++ b/processMeerKAT/DEFAULTS.cfg @@ -0,0 +1,145 @@ +[ilifu] + # Set global limits for current ilifu cluster configuration + TOTAL_NODES_LIMIT = 79 + CPUS_PER_NODE_LIMIT = 32 + NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s + MEM_PER_NODE_GB_LIMIT = 232 #237568 MB + MEM_PER_NODE_GB_LIMIT_HIGHMEM = 480 #491520 MB + ACCOUNTS = ['b03-idia-ag','b05-pipelines-ag'] + + # Set global values for paths and file names + LOG_DIR = 'logs' + CALIB_SCRIPTS_DIR = 'crosscal_scripts' + AUX_SCRIPTS_DIR = 'aux_scripts' + SELFCAL_SCRIPTS_DIR = 'selfcal_scripts' + CONFIG = 'default_config.txt' + TMP_CONFIG = '.config.tmp' + MASTER_SCRIPT = 'submit_pipeline.sh' + + # Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values + FIELDS_CONFIG_KEYS = [ + 'fluxfield','bpassfield','phasecalfield', + 'targetfields','extrafields' + ] + CROSSCAL_CONFIG_KEYS = [ + 'minbaselines','chanbin','width','timeavg','createmms])', + 'keepmms','spw','nspw','calcrefant','refant','standard', + 'badants','badfreqranges' + ] + SELFCAL_CONFIG_KEYS = [ + 'nloops','loop','cell','robust','imsize','wprojplanes', + 'niter','threshold','uvrange','nterms','gridder','deconvolver', + 'solint','calmode','discard_nloops','gaintype', + 'outlier_threshold','flag' + ] + IMAGING_CONFIG_KEYS = [ + 'cell', 'robust', 'imsize', 'wprojplanes', 'niter', + 'threshold', 'multiscale', 'nterms', 'gridder', + 'deconvolver', 'restoringbeam', 'specmode', + 'stokes', 'mask', 'rmsmap' + ] + SLURM_CONFIG_STR_KEYS = [ + 'container','mpi_wrapper','partition','time','name', + 'dependencies','exclude','account','reservation' + ] + SLURM_CONFIG_KEYS_BASE = [ + 'nodes','ntasks_per_node','mem','plane','submit', + 'precal_scripts','postcal_scripts','scripts', + 'verbose','modules' + ] + CONTAINER = "/idia/software/containers/casa-6.3.simg" + MPI_WRAPPER = 'mpirun' + PRECAL_SCRIPTS = [('calc_refant.py',False,''),('partition.py',True,'')] #Scripts run before calibration at top level directory when nspw > 1 + POSTCAL_SCRIPTS = [ + ('concat.py',False,''), + ('plotcal_spw.py', False, ''), + ('selfcal_part1.py',True,''), + ('selfcal_part2.py',False,''), + ('science_image.py', True, '') + ] #Scripts run after calibration at top level directory when nspw > 1 + SCRIPTS = [ + ('validate_input.py',False,''), + ('flag_round_1.py',True,''), + ('calc_refant.py',False,''), + ('setjy.py',True,''), + ('xx_yy_solve.py',False,''), + ('xx_yy_apply.py',True,''), + ('flag_round_2.py',True,''), + ('xx_yy_solve.py',False,''), + ('xx_yy_apply.py',True,''), + ('split.py',True,''), + ('quick_tclean.py',True,'') + ] + +[galahad] + # Set global limits for current ilifu cluster configuration + TOTAL_NODES_LIMIT = 1 ### update me! + CPUS_PER_NODE_LIMIT = 16 + NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s + MEM_PER_NODE_GB_LIMIT = 1000 #237568 MB ### update me!!! + MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1300 #491520 MB ### update me!!! + ACCOUNTS = [] ### update me!!! + + # Set global values for paths and file names + LOG_DIR = 'logs' + CALIB_SCRIPTS_DIR = 'crosscal_scripts' + AUX_SCRIPTS_DIR = 'aux_scripts' + SELFCAL_SCRIPTS_DIR = 'selfcal_scripts' + CONFIG = 'default_config.txt' + TMP_CONFIG = '.config.tmp' + MASTER_SCRIPT = 'submit_pipeline.sh' + + # Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values + FIELDS_CONFIG_KEYS = [ + 'fluxfield','bpassfield','phasecalfield', + 'targetfields','extrafields' + ] + CROSSCAL_CONFIG_KEYS = [ + 'minbaselines','chanbin','width','timeavg','createmms', + 'keepmms','spw','nspw','calcrefant','refant','standard', + 'badants','badfreqranges' + ] + SELFCAL_CONFIG_KEYS = [ + 'nloops','loop','cell','robust','imsize','wprojplanes', + 'niter','threshold','uvrange','nterms','gridder','deconvolver', + 'solint','calmode','discard_nloops','gaintype', + 'outlier_threshold','flag' + ] + IMAGING_CONFIG_KEYS = [ + 'cell', 'robust', 'imsize', 'wprojplanes', 'niter', + 'threshold', 'multiscale', 'nterms', 'gridder', + 'deconvolver', 'restoringbeam', 'specmode', + 'stokes', 'mask', 'rmsmap' + ] + SLURM_CONFIG_STR_KEYS = [ + 'container','mpi_wrapper','partition','time','name', + 'dependencies','exclude','account','reservation' + ] + SLURM_CONFIG_KEYS_BASE = [ + 'nodes','ntasks_per_node','mem','plane','submit', + 'precal_scripts','postcal_scripts','scripts', + 'verbose','modules' + ] + CONTAINER = '/share/nas/mbowles/mightee/casa-stable.simg' + MPI_WRAPPER = %(CONTAINER)s # Fairly certain this shouldnt work, but it might. I.e. should probably still reference mpirun somehow. + PRECAL_SCRIPTS = [('calc_refant.py',False,''),('partition.py',True,'')] #Scripts run before calibration at top level directory when nspw > 1 + POSTCAL_SCRIPTS = [ + ('concat.py',False,''), + ('plotcal_spw.py', False, ''), + ('selfcal_part1.py',True,''), + ('selfcal_part2.py',False,''), + ('science_image.py', True, '') + ] #Scripts run after calibration at top level directory when nspw > 1 + SCRIPTS = [ + ('validate_input.py',False,''), + ('flag_round_1.py',True,''), + ('calc_refant.py',False,''), + ('setjy.py',True,''), + ('xx_yy_solve.py',False,''), + ('xx_yy_apply.py',True,''), + ('flag_round_2.py',True,''), + ('xx_yy_solve.py',False,''), + ('xx_yy_apply.py',True,''), + ('split.py',True,''), + ('quick_tclean.py',True,'') + ] diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index e8eff6d..c6ac212 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -36,46 +36,6 @@ logger = logging.getLogger(__name__) logging.basicConfig(format="%(asctime)-15s %(levelname)s: %(message)s") -#Set global limits for current ilifu cluster configuration -TOTAL_NODES_LIMIT = 79 -CPUS_PER_NODE_LIMIT = 32 -NTASKS_PER_NODE_LIMIT = CPUS_PER_NODE_LIMIT -MEM_PER_NODE_GB_LIMIT = 232 #237568 MB -MEM_PER_NODE_GB_LIMIT_HIGHMEM = 480 #491520 MB - -#Set global values for paths and file names -THIS_PROG = __file__ -SCRIPT_DIR = os.path.dirname(THIS_PROG) -LOG_DIR = 'logs' -CALIB_SCRIPTS_DIR = 'crosscal_scripts' -AUX_SCRIPTS_DIR = 'aux_scripts' -SELFCAL_SCRIPTS_DIR = 'selfcal_scripts' -CONFIG = 'default_config.txt' -TMP_CONFIG = '.config.tmp' -MASTER_SCRIPT = 'submit_pipeline.sh' - -#Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values -FIELDS_CONFIG_KEYS = ['fluxfield','bpassfield','phasecalfield','targetfields','extrafields'] -CROSSCAL_CONFIG_KEYS = ['minbaselines','chanbin','width','timeavg','createmms','keepmms','spw','nspw','calcrefant','refant','standard','badants','badfreqranges'] -SELFCAL_CONFIG_KEYS = ['nloops','loop','cell','robust','imsize','wprojplanes','niter','threshold','uvrange','nterms','gridder','deconvolver','solint','calmode','discard_nloops','gaintype','outlier_threshold','flag'] -IMAGING_CONFIG_KEYS = ['cell', 'robust', 'imsize', 'wprojplanes', 'niter', 'threshold', 'multiscale', 'nterms', 'gridder', 'deconvolver', 'restoringbeam', 'specmode', 'stokes', 'mask', 'rmsmap'] -SLURM_CONFIG_STR_KEYS = ['container','mpi_wrapper','partition','time','name','dependencies','exclude','account','reservation'] -SLURM_CONFIG_KEYS = ['nodes','ntasks_per_node','mem','plane','submit','precal_scripts','postcal_scripts','scripts','verbose','modules'] + SLURM_CONFIG_STR_KEYS -CONTAINER = '/idia/software/containers/casa-6.3.simg' -MPI_WRAPPER = 'mpirun' -PRECAL_SCRIPTS = [('calc_refant.py',False,''),('partition.py',True,'')] #Scripts run before calibration at top level directory when nspw > 1 -POSTCAL_SCRIPTS = [('concat.py',False,''),('plotcal_spw.py', False, ''),('selfcal_part1.py',True,''),('selfcal_part2.py',False,''),('science_image.py', True, '')] #Scripts run after calibration at top level directory when nspw > 1 -SCRIPTS = [ ('validate_input.py',False,''), - ('flag_round_1.py',True,''), - ('calc_refant.py',False,''), - ('setjy.py',True,''), - ('xx_yy_solve.py',False,''), - ('xx_yy_apply.py',True,''), - ('flag_round_2.py',True,''), - ('xx_yy_solve.py',False,''), - ('xx_yy_apply.py',True,''), - ('split.py',True,''), - ('quick_tclean.py',True,'')] def check_path(path,update=False): @@ -172,7 +132,7 @@ def parse_scripts(val): parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) - parser.add_argument("--cluster",metavar='name', required=False, type=str, default="ilifu", help="Name of cluster being used [default: ilifu; allowed: galahad, ilifu]") + parser.add_argument("--cluster",metavar='name', required=False, type=str, default="ilifu", help="Name of cluster being used if not ilifu default slurm limits are removed [default: ilifu].") parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") parser.add_argument("-C","--config",metavar="path", default=CONFIG, required=False, type=str, help="Relative (not absolute) path to config file.") parser.add_argument("-N","--nodes",metavar="num", required=False, type=int, default=1, @@ -271,10 +231,6 @@ def validate_args(args,config,parser=None): parser : class ``argparse.ArgumentParser``, optional If this is input, parser error will be raised.""" - if args['cluster'] not in ['ilifu','galahad']: - msg = "The selected cluster must be one of [ilifu, galahad]. Pipeline has not been implemented for other clusters yet." - raise_error(config, msg, parser) - if parser is None or args['build']: if args['MS'] is None and not args['nofields']: msg = "You must input an MS [-M --MS] to build the config file." @@ -288,57 +244,62 @@ def validate_args(args,config,parser=None): msg = "Only input an MS [-M --MS] during [-B --build] step. Otherwise input is ignored." raise_error(config, msg, parser) - if args['ntasks_per_node'] > NTASKS_PER_NODE_LIMIT: - msg = "The number of tasks per node [-t --ntasks-per-node] must not exceed {0}. You input {1}.".format(NTASKS_PER_NODE_LIMIT,args['ntasks_per_node']) - raise_error(config, msg, parser) - - if args['nodes'] > TOTAL_NODES_LIMIT: - msg = "The number of nodes [-N --nodes] per node must not exceed {0}. You input {1}.".format(TOTAL_NODES_LIMIT,args['nodes']) - raise_error(config, msg, parser) + if args['cluster'] not in SAFE_CLUSTERS: + msg = "Specified cluster [--cluster] is not in {0}. You input {1}. Pipeline will rely entirely on the specified config. No upper limits will be set. HPC specific selections within your config which are not actually availble may cause pipeline runs to fail!" + logger.warning(msg.format(SAFE_CLUSTERS, args['cluster'])) - if args['mem'] > MEM_PER_NODE_GB_LIMIT: - if args['partition'] != 'HighMem': - msg = "The memory per node [-m --mem] must not exceed {0} (GB). You input {1} (GB).".format(MEM_PER_NODE_GB_LIMIT,args['mem']) - raise_error(config, msg, parser) - elif args['mem'] > MEM_PER_NODE_GB_LIMIT_HIGHMEM: - msg = "The memory per node [-m --mem] must not exceed {0} (GB) when using 'HighMem' partition. You input {1} (GB).".format(MEM_PER_NODE_GB_LIMIT_HIGHMEM,args['mem']) + else: + if args['ntasks_per_node'] > NTASKS_PER_NODE_LIMIT: + msg = "The number of tasks per node [-t --ntasks-per-node] must not exceed {0}. You input {1}.".format(NTASKS_PER_NODE_LIMIT,args['ntasks_per_node']) raise_error(config, msg, parser) - if args['plane'] > args['ntasks_per_node']: - msg = "The value of [-P --plane] cannot be greater than the tasks per node [-t --ntasks-per-node] ({0}). You input {1}.".format(args['ntasks_per_node'],args['plane']) - raise_error(config, msg, parser) + if args['nodes'] > TOTAL_NODES_LIMIT: + msg = "The number of nodes [-N --nodes] per node must not exceed {0}. You input {1}.".format(TOTAL_NODES_LIMIT,args['nodes']) + raise_error(config, msg, parser) - if args['account'] not in ['b03-idia-ag','b05-pipelines-ag']: - from platform import node - if 'slurm-login' in node() or 'slwrk' in node() or 'compute' in node(): - accounts=os.popen("for f in $(sacctmgr show user $USER --noheader cluster=ilifu-slurm20 -s format=account%30); do echo -n $f,; done").read()[:-1].split(',') - if args['account'] not in accounts: - msg = "Accounting group '{0}' not recognised. Please select one of the following from your groups: {1}.".format(args['account'],accounts) - for account in accounts: - if args['account'] in account: - msg += ' Perhaps you meant accounting group "{0}".'.format(account) - break + if args['mem'] > MEM_PER_NODE_GB_LIMIT: + if args['partition'] != 'HighMem': + msg = "The memory per node [-m --mem] must not exceed {0} (GB). You input {1} (GB).".format(MEM_PER_NODE_GB_LIMIT,args['mem']) raise_error(config, msg, parser) - else: - msg = "Accounting group '{0}' not recognised. You're not using a SLURM node, so cannot query your accounts.".format(args['account']) + elif args['mem'] > MEM_PER_NODE_GB_LIMIT_HIGHMEM: + msg = "The memory per node [-m --mem] must not exceed {0} (GB) when using 'HighMem' partition. You input {1} (GB).".format(MEM_PER_NODE_GB_LIMIT_HIGHMEM,args['mem']) + raise_error(config, msg, parser) + + if args['plane'] > args['ntasks_per_node']: + msg = "The value of [-P --plane] cannot be greater than the tasks per node [-t --ntasks-per-node] ({0}). You input {1}.".format(args['ntasks_per_node'],args['plane']) raise_error(config, msg, parser) - if args['reservation'] != '': - from platform import node - if 'slurm-login' in node() or 'slwrk' in node() or 'compute' in node(): - reservations=os.popen("scontrol show reservation | grep ReservationName | awk '{print $1}' | cut -d = -f2").read()[:-1].split('\n') - if args['reservation'] not in reservations: - msg = "Reservation '{0}' not recognised.".format(args['reservation']) - if reservations == ['']: - msg += ' There are no active reservations.' - else: - msg += ' Please select one of the following reservations, if applicable: {0}.'.format(reservations) + if args['account'] not in ACCOUNTS: + from platform import node + if 'slurm-login' in node() or 'slwrk' in node() or 'compute' in node(): + accounts=os.popen("for f in $(sacctmgr show user $USER --noheader cluster=ilifu-slurm20 -s format=account%30); do echo -n $f,; done").read()[:-1].split(',') + if args['account'] not in accounts: + msg = "Accounting group '{0}' not recognised. Please select one of the following from your groups: {1}.".format(args['account'],accounts) + for account in accounts: + if args['account'] in account: + msg += ' Perhaps you meant accounting group "{0}".'.format(account) + break + raise_error(config, msg, parser) + else: + msg = "Accounting group '{0}' not recognised. You're not using a SLURM node, so cannot query your accounts.".format(args['account']) + raise_error(config, msg, parser) + + if args['reservation'] != '': + from platform import node + if 'slurm-login' in node() or 'slwrk' in node() or 'compute' in node(): + reservations=os.popen("scontrol show reservation | grep ReservationName | awk '{print $1}' | cut -d = -f2").read()[:-1].split('\n') + if args['reservation'] not in reservations: + msg = "Reservation '{0}' not recognised.".format(args['reservation']) + if reservations == ['']: + msg += ' There are no active reservations.' + else: + msg += ' Please select one of the following reservations, if applicable: {0}.'.format(reservations) + raise_error(config, msg, parser) + else: + msg = "Reservation '{0}' not recognised. You're not using a SLURM node, so cannot query your accounts.".format(args['reservation']) raise_error(config, msg, parser) - else: - msg = "Reservation '{0}' not recognised. You're not using a SLURM node, so cannot query your accounts.".format(args['reservation']) - raise_error(config, msg, parser) -def write_command(script,args,name='job',mpi_wrapper=MPI_WRAPPER,container=CONTAINER,casa_script=False,logfile=True,plot=False,SPWs='',nspw=1): +def write_command(script,args,name='job',mpi_wrapper=MPI_WRAPPER,container='casa.simg',casa_script=False,logfile=True,plot=False,SPWs='',nspw=1): """Write bash command to call a script (with args) directly with srun, or within sbatch file, optionally via CASA. @@ -410,7 +371,7 @@ def write_command(script,args,name='job',mpi_wrapper=MPI_WRAPPER,container=CONTA return command -def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="job",runname='',plane=1,exclude='',mpi_wrapper=MPI_WRAPPER,container=CONTAINER, +def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="job",runname='',plane=1,exclude='',mpi_wrapper=MPI_WRAPPER,container='casa.simg', partition="Main",time="12:00:00",casa_script=False,SPWs='',nspw=1,account='b03-idia-ag',reservation='',modules=[],justrun=False): """Write a SLURM sbatch file calling a certain script (and args) with a particular configuration. @@ -1527,58 +1488,55 @@ def setup_logger(config,verbose=False): def main(): + # Define defaults / limits for named HPC facilities + THIS_PROG = __file__ + SCRIPT_DIR = os.path.dirname(THIS_PROG) + DEFAULTS_CONFIG_PATH = "DEFAULTS.cfg" + CONFIG_DEFAULTS,_ = config_parser.parse_config( + "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) + ) + KNOWN_CLUSTERS = DEFAULTS.keys() + + # TO DO + #Parse command-line arguments, and setup logger args = parse_args() setup_logger(args.config,args.verbose) - # Cluster adaptations if required - if args.cluser=='galahad': - print('Configuring pipeline for use on Galahad ...') - args['partition']='WHEEL' - args['mem']=1000 - MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1000 - MEM_PER_NODE_GB_LIMIT = 1000 - # Set global limits for current ilifu cluster configuration - TOTAL_NODES_LIMIT = 1 - CPUS_PER_NODE_LIMIT = 16 - NTASKS_PER_NODE_LIMIT = CPUS_PER_NODE_LIMIT - MEM_PER_NODE_GB_LIMIT = 1000 #237568 MB - MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1300 #491520 MB - - # Set global values for paths and file names - THIS_PROG = __file__ - SCRIPT_DIR = os.path.dirname(THIS_PROG) - LOG_DIR = 'logs' - CALIB_SCRIPTS_DIR = 'crosscal_scripts' - AUX_SCRIPTS_DIR = 'aux_scripts' - SELFCAL_SCRIPTS_DIR = 'selfcal_scripts' - CONFIG = 'default_config.txt' - TMP_CONFIG = '.config.tmp' - MASTER_SCRIPT = 'submit_pipeline.sh' - - #Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values - FIELDS_CONFIG_KEYS = ['fluxfield','bpassfield','phasecalfield','targetfields','extrafields'] - CROSSCAL_CONFIG_KEYS = ['minbaselines','chanbin','width','timeavg','createmms','keepmms','spw','nspw','calcrefant','refant','standard','badants','badfreqranges'] - SELFCAL_CONFIG_KEYS = ['nloops','restart_no','cell','robust','imsize','wprojplanes','niter','threshold','multiscale','nterms','gridder','deconvolver','solint','calmode','atrous'] - SLURM_CONFIG_STR_KEYS = ['container','mpi_wrapper','partition','time','name','dependencies','exclude','account','reservation'] - SLURM_CONFIG_KEYS = ['nodes','ntasks_per_node','mem','plane','submit','precal_scripts','postcal_scripts','scripts','verbose'] + SLURM_CONFIG_STR_KEYS - CONTAINER = '/share/nas/mbowles/mightee/casa-stable.simg' - MPI_WRAPPER = CONTAINER # Fairly certain this shouldnt work, but it might. - PRECAL_SCRIPTS = [('calc_refant.py',False,''),('partition.py',True,'')] #Scripts run before calibration at top level directory when nspw > 1 - POSTCAL_SCRIPTS = [('concat.py',False,''),('plotcal_spw.py', False, ''),('selfcal_part1.py',True,''),('selfcal_part2.py',False,''),('run_bdsf.py', False, ''),('make_pixmask.py', False, '')] #Scripts run after calibration at top level directory when nspw > 1 - SCRIPTS = [ ('validate_input.py',False,''), - ('flag_round_1.py',True,''), - ('calc_refant.py',False,''), - ('setjy.py',True,''), - ('xx_yy_solve.py',False,''), - ('xx_yy_apply.py',True,''), - ('flag_round_2.py',True,''), - ('xx_yy_solve.py',False,''), - ('xx_yy_apply.py',True,''), - ('split.py',True,''), - ('quick_tclean.py',True,''), - ('plot_solutions.py',False,'')] - + # Select default source + if args.cluster in SAFE_CLUSTERS: + DEFAULTS = CONFIG_DEFAULTS[args.cluster] + else: ### Decide what to do when the cluster is not specified. + pass + # Set global limits for current ilifu cluster configuration + TOTAL_NODES_LIMIT = DEFAULTS['TOTAL_NODES_LIMIT'] + CPUS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'] + NTASKS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'] + MEM_PER_NODE_GB_LIMIT = DEFAULTS['MEM_PER_NODE_GB_LIMIT'] #237568 MB + MEM_PER_NODE_GB_LIMIT_HIGHMEM = DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'] #491520 MB + ACCOUNTS = DEFAULTS['ACCOUNTS'] + + #Set global values for paths and file names + LOG_DIR = DEFAULTS['LOG_DIR'] + CALIB_SCRIPTS_DIR = DEFAULTS['CALIB_SCRIPTS_DIR'] + AUX_SCRIPTS_DIR = DEFAULTS['AUX_SCRIPTS_DIR'] + SELFCAL_SCRIPTS_DIR = DEFAULTS['SELFCAL_SCRIPTS_DIR'] + CONFIG = DEFAULTS['CONFIG'] + TMP_CONFIG = DEFAULTS['TMP_CONFIG'] + MASTER_SCRIPT = DEFAULTS['MASTER_SCRIPT'] + + #Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values + FIELDS_CONFIG_KEYS = DEFAULTS['FIELDS_CONFIG_KEYS'] + CROSSCAL_CONFIG_KEYS = DEFAULTS['CROSSCAL_CONFIG_KEYS'] + SELFCAL_CONFIG_KEYS = DEFAULTS['SELFCAL_CONFIG_KEYS'] + IMAGING_CONFIG_KEYS = DEFAULTS['IMAGING_CONFIG_KEYS'] + SLURM_CONFIG_STR_KEYS = DEFAULTS['SLURM_CONFIG_KEYS'] + SLURM_CONFIG_KEYS = DEFAULTS['SLURM_CONFIG_KEYS_BASE'] + SLURM_CONFIG_STR_KEYS + CONTAINER = DEFAULTS['CONTAINER'] + MPI_WRAPPER = DEFAULTS['MPI_WRAPPER'] + PRECAL_SCRIPTS = DEFAULTS['PRECAL_SCRIPTS'] + POSTCAL_SCRIPTS = DEFAULTS['POSTCAL_SCRIPTS'] + SCRIPTS = DEFAULTS['SCRIPTS'] #Mutually exclusive arguments - display version, build config file or run pipeline if args.version: From 0f64df1395c0445f6e378fc5065955c6b595e21e Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 4 Oct 2021 15:56:35 +0100 Subject: [PATCH 04/39] Removing globals from function definitions and cleaning DEFAULTS.cfg --- processMeerKAT/DEFAULTS.cfg | 71 +++----------------------------- processMeerKAT/processMeerKAT.py | 54 ++++++++++++------------ 2 files changed, 32 insertions(+), 93 deletions(-) diff --git a/processMeerKAT/DEFAULTS.cfg b/processMeerKAT/DEFAULTS.cfg index d87923a..e849c92 100644 --- a/processMeerKAT/DEFAULTS.cfg +++ b/processMeerKAT/DEFAULTS.cfg @@ -1,4 +1,4 @@ -[ilifu] +[DEFAULT] # Set global limits for current ilifu cluster configuration TOTAL_NODES_LIMIT = 79 CPUS_PER_NODE_LIMIT = 32 @@ -71,75 +71,16 @@ ('quick_tclean.py',True,'') ] +[ilifu] + # [DEFAULTS] section defined at the top of this file represents the ilifu configuration. + [galahad] - # Set global limits for current ilifu cluster configuration + # Specify differences to ilifu TOTAL_NODES_LIMIT = 1 ### update me! CPUS_PER_NODE_LIMIT = 16 NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s MEM_PER_NODE_GB_LIMIT = 1000 #237568 MB ### update me!!! MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1300 #491520 MB ### update me!!! ACCOUNTS = [] ### update me!!! - - # Set global values for paths and file names - LOG_DIR = 'logs' - CALIB_SCRIPTS_DIR = 'crosscal_scripts' - AUX_SCRIPTS_DIR = 'aux_scripts' - SELFCAL_SCRIPTS_DIR = 'selfcal_scripts' - CONFIG = 'default_config.txt' - TMP_CONFIG = '.config.tmp' - MASTER_SCRIPT = 'submit_pipeline.sh' - - # Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values - FIELDS_CONFIG_KEYS = [ - 'fluxfield','bpassfield','phasecalfield', - 'targetfields','extrafields' - ] - CROSSCAL_CONFIG_KEYS = [ - 'minbaselines','chanbin','width','timeavg','createmms', - 'keepmms','spw','nspw','calcrefant','refant','standard', - 'badants','badfreqranges' - ] - SELFCAL_CONFIG_KEYS = [ - 'nloops','loop','cell','robust','imsize','wprojplanes', - 'niter','threshold','uvrange','nterms','gridder','deconvolver', - 'solint','calmode','discard_nloops','gaintype', - 'outlier_threshold','flag' - ] - IMAGING_CONFIG_KEYS = [ - 'cell', 'robust', 'imsize', 'wprojplanes', 'niter', - 'threshold', 'multiscale', 'nterms', 'gridder', - 'deconvolver', 'restoringbeam', 'specmode', - 'stokes', 'mask', 'rmsmap' - ] - SLURM_CONFIG_STR_KEYS = [ - 'container','mpi_wrapper','partition','time','name', - 'dependencies','exclude','account','reservation' - ] - SLURM_CONFIG_KEYS_BASE = [ - 'nodes','ntasks_per_node','mem','plane','submit', - 'precal_scripts','postcal_scripts','scripts', - 'verbose','modules' - ] CONTAINER = '/share/nas/mbowles/mightee/casa-stable.simg' - MPI_WRAPPER = %(CONTAINER)s # Fairly certain this shouldnt work, but it might. I.e. should probably still reference mpirun somehow. - PRECAL_SCRIPTS = [('calc_refant.py',False,''),('partition.py',True,'')] #Scripts run before calibration at top level directory when nspw > 1 - POSTCAL_SCRIPTS = [ - ('concat.py',False,''), - ('plotcal_spw.py', False, ''), - ('selfcal_part1.py',True,''), - ('selfcal_part2.py',False,''), - ('science_image.py', True, '') - ] #Scripts run after calibration at top level directory when nspw > 1 - SCRIPTS = [ - ('validate_input.py',False,''), - ('flag_round_1.py',True,''), - ('calc_refant.py',False,''), - ('setjy.py',True,''), - ('xx_yy_solve.py',False,''), - ('xx_yy_apply.py',True,''), - ('flag_round_2.py',True,''), - ('xx_yy_solve.py',False,''), - ('xx_yy_apply.py',True,''), - ('split.py',True,''), - ('quick_tclean.py',True,'') - ] + MPI_WRAPPER = %(CONTAINER)s ### update me!!! # Fairly certain this shouldnt work, but it might. I.e. should probably still reference mpirun somehow. diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index c6ac212..6bb2ab6 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -244,9 +244,9 @@ def validate_args(args,config,parser=None): msg = "Only input an MS [-M --MS] during [-B --build] step. Otherwise input is ignored." raise_error(config, msg, parser) - if args['cluster'] not in SAFE_CLUSTERS: - msg = "Specified cluster [--cluster] is not in {0}. You input {1}. Pipeline will rely entirely on the specified config. No upper limits will be set. HPC specific selections within your config which are not actually availble may cause pipeline runs to fail!" - logger.warning(msg.format(SAFE_CLUSTERS, args['cluster'])) + if args['cluster'] not in KNOWN_CLUSTERS: + msg = "Cluster [--cluster] is not in {0}. You input {1}. Pipeline will rely entirely on the specified config. No upper limits will be set. HPC specific selections within your config which are not actually availble may cause pipeline runs to fail!" + logger.warning(msg.format(KNOWN_CLUSTERS, args['cluster'])) else: if args['ntasks_per_node'] > NTASKS_PER_NODE_LIMIT: @@ -299,7 +299,7 @@ def validate_args(args,config,parser=None): msg = "Reservation '{0}' not recognised. You're not using a SLURM node, so cannot query your accounts.".format(args['reservation']) raise_error(config, msg, parser) -def write_command(script,args,name='job',mpi_wrapper=MPI_WRAPPER,container='casa.simg',casa_script=False,logfile=True,plot=False,SPWs='',nspw=1): +def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False,logfile=True,plot=False,SPWs='',nspw=1): """Write bash command to call a script (with args) directly with srun, or within sbatch file, optionally via CASA. @@ -309,12 +309,12 @@ def write_command(script,args,name='job',mpi_wrapper=MPI_WRAPPER,container='casa Path to script called (assumed to exist or be in PATH or calibration scripts directory). args : str Arguments to pass into script. Use '' for no arguments. - name : str, optional - Name of this job, to append to CASA output name. - mpi_wrapper : str, optional + mpi_wrapper : str MPI wrapper for this job. e.g. 'srun', 'mpirun', 'mpicasa' (may need to specify path). - container : str, optional + container : str Path to singularity container used for this job. + name : str, optional + Name of this job, to append to CASA output name. casa_script : bool, optional Is the script that is called within this job a CASA script? logfile : bool, optional @@ -371,7 +371,7 @@ def write_command(script,args,name='job',mpi_wrapper=MPI_WRAPPER,container='casa return command -def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="job",runname='',plane=1,exclude='',mpi_wrapper=MPI_WRAPPER,container='casa.simg', +def write_sbatch(script,args,mem,mpi_wrapper,nodes=1,tasks=16,name="job",runname='',plane=1,exclude='',container='casa.simg', partition="Main",time="12:00:00",casa_script=False,SPWs='',nspw=1,account='b03-idia-ag',reservation='',modules=[],justrun=False): """Write a SLURM sbatch file calling a certain script (and args) with a particular configuration. @@ -381,14 +381,16 @@ def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="jo Path to script called within sbatch file (assumed to exist or be in PATH or calibration directory). args : str Arguments passed into script called within this sbatch file. Use '' for no arguments. + mem : int + The memory in GB (per node) to use for this job. + mpi_wrapper : str + MPI wrapper for this job. e.g. 'srun', 'mpirun', 'mpicasa' (may need to specify path). time : str, optional Time limit on this job. nodes : int, optional Number of nodes to use for this job. tasks : int, optional The number of tasks per node to use for this job. - mem : int, optional - The memory in GB (per node) to use for this job. name : str, optional Name for this job, used in naming the various output files. runname : str, optional @@ -397,8 +399,6 @@ def write_sbatch(script,args,nodes=1,tasks=16,mem=MEM_PER_NODE_GB_LIMIT,name="jo Distrubute tasks for this job using this block size before moving onto next node. exclude : str, optional SLURM worker nodes to exclude. - mpi_wrapper : str, optional - MPI wrapper for this job. e.g. 'srun', 'mpirun', 'mpicasa' (may need to specify path). container : str, optional Path to singularity container used for this job. partition : str, optional @@ -907,7 +907,7 @@ def srun(arg_dict,qos=True,time=10,mem=4): return call -def write_jobs(config, scripts=[], threadsafe=[], containers=[], num_precal_scripts=0, mpi_wrapper=MPI_WRAPPER, nodes=8, ntasks_per_node=4, mem=MEM_PER_NODE_GB_LIMIT,plane=1, partition='Main', +def write_jobs(config, mpi_wrapper, mem, scripts=[], threadsafe=[], containers=[], num_precal_scripts=0, nodes=8, ntasks_per_node=4, plane=1, partition='Main', time='12:00:00', submit=False, name='', verbose=False, quiet=False, dependencies='', exclude='', account='b03-idia-ag', reservation='', modules=[], timestamp='', justrun=False): """Write a series of sbatch job files to calibrate a CASA MeasurementSet. @@ -916,6 +916,10 @@ def write_jobs(config, scripts=[], threadsafe=[], containers=[], num_precal_scri ---------- config : str Path to config file. + mem : int + The memory in GB (per node) to use for this job. + mpi_wrapper : str + Path to MPI wrapper to use for threadsafe tasks (otherwise srun used). scripts : list (of paths), optional List of paths to scripts (assumed to be python -- i.e. extension .py) to call within seperate sbatch jobs. threadsafe : list (of bools), optional @@ -924,14 +928,10 @@ def write_jobs(config, scripts=[], threadsafe=[], containers=[], num_precal_scri List of paths to singularity containers to use for each script. List assumed to be same length as scripts. num_precal_scripts : int, optional Number of precal scripts. - mpi_wrapper : str, optional - Path to MPI wrapper to use for threadsafe tasks (otherwise srun used). nodes : int, optional Number of nodes to use for this job. tasks : int, optional The number of tasks per node to use for this job. - mem : int, optional - The memory in GB (per node) to use for this job. plane : int, optional Distrubute tasks for this job using this block size before moving onto next node. partition : str, optional @@ -1497,26 +1497,24 @@ def main(): ) KNOWN_CLUSTERS = DEFAULTS.keys() - # TO DO - #Parse command-line arguments, and setup logger args = parse_args() setup_logger(args.config,args.verbose) # Select default source - if args.cluster in SAFE_CLUSTERS: + if args.cluster in KNOWN_CLUSTERS: DEFAULTS = CONFIG_DEFAULTS[args.cluster] else: ### Decide what to do when the cluster is not specified. pass - # Set global limits for current ilifu cluster configuration + # Set limits for current cluster configuration TOTAL_NODES_LIMIT = DEFAULTS['TOTAL_NODES_LIMIT'] CPUS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'] NTASKS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'] - MEM_PER_NODE_GB_LIMIT = DEFAULTS['MEM_PER_NODE_GB_LIMIT'] #237568 MB - MEM_PER_NODE_GB_LIMIT_HIGHMEM = DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'] #491520 MB + MEM_PER_NODE_GB_LIMIT = DEFAULTS['MEM_PER_NODE_GB_LIMIT'] + MEM_PER_NODE_GB_LIMIT_HIGHMEM = DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'] ACCOUNTS = DEFAULTS['ACCOUNTS'] - #Set global values for paths and file names + # Set global values for paths and file names LOG_DIR = DEFAULTS['LOG_DIR'] CALIB_SCRIPTS_DIR = DEFAULTS['CALIB_SCRIPTS_DIR'] AUX_SCRIPTS_DIR = DEFAULTS['AUX_SCRIPTS_DIR'] @@ -1525,7 +1523,7 @@ def main(): TMP_CONFIG = DEFAULTS['TMP_CONFIG'] MASTER_SCRIPT = DEFAULTS['MASTER_SCRIPT'] - #Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values + # Set global values for field, crosscal and SLURM arguments copied to config file FIELDS_CONFIG_KEYS = DEFAULTS['FIELDS_CONFIG_KEYS'] CROSSCAL_CONFIG_KEYS = DEFAULTS['CROSSCAL_CONFIG_KEYS'] SELFCAL_CONFIG_KEYS = DEFAULTS['SELFCAL_CONFIG_KEYS'] @@ -1538,7 +1536,7 @@ def main(): POSTCAL_SCRIPTS = DEFAULTS['POSTCAL_SCRIPTS'] SCRIPTS = DEFAULTS['SCRIPTS'] - #Mutually exclusive arguments - display version, build config file or run pipeline + # Mutually exclusive arguments - display version, build config file or run pipeline if args.version: logger.info('This is version {0}'.format(__version__)) if args.license: @@ -1547,7 +1545,7 @@ def main(): default_config(vars(args)) if args.run: kwargs = format_args(args.config,args.submit,args.quiet,args.dependencies,args.justrun) - write_jobs(args.config, **kwargs) + write_jobs(args.config, mpi_wrapper=MPI_WRAPPER,**kwargs) if __name__ == "__main__": main() From 7406bac701a4796322c11812e6721766b14567a5 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 4 Oct 2021 16:44:07 +0100 Subject: [PATCH 05/39] HPC specific SBATCH file header implementation. --- processMeerKAT/DEFAULTS.cfg | 4 ++ processMeerKAT/processMeerKAT.py | 100 +++++++++++-------------------- 2 files changed, 39 insertions(+), 65 deletions(-) diff --git a/processMeerKAT/DEFAULTS.cfg b/processMeerKAT/DEFAULTS.cfg index e849c92..619b79c 100644 --- a/processMeerKAT/DEFAULTS.cfg +++ b/processMeerKAT/DEFAULTS.cfg @@ -70,6 +70,8 @@ ('split.py',True,''), ('quick_tclean.py',True,'') ] + # sbatch_file_base must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. + sbatch_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --account={account}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --partition={partition}\n#SBATCH --time={time}" [ilifu] # [DEFAULTS] section defined at the top of this file represents the ilifu configuration. @@ -84,3 +86,5 @@ ACCOUNTS = [] ### update me!!! CONTAINER = '/share/nas/mbowles/mightee/casa-stable.simg' MPI_WRAPPER = %(CONTAINER)s ### update me!!! # Fairly certain this shouldnt work, but it might. I.e. should probably still reference mpirun somehow. + # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. + sbatch_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}" diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 6bb2ab6..c2d282e 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -371,7 +371,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False return command -def write_sbatch(script,args,mem,mpi_wrapper,nodes=1,tasks=16,name="job",runname='',plane=1,exclude='',container='casa.simg', +def write_sbatch(script,args,mem,mpi_wrapper,contents,nodes=1,tasks=16,name="job",runname='',plane=1,exclude='',container='casa.simg', partition="Main",time="12:00:00",casa_script=False,SPWs='',nspw=1,account='b03-idia-ag',reservation='',modules=[],justrun=False): """Write a SLURM sbatch file calling a certain script (and args) with a particular configuration. @@ -480,39 +480,9 @@ def write_sbatch(script,args,mem,mpi_wrapper,nodes=1,tasks=16,name="job",runname for module in modules: if len(module) > 0: params['modules'] += "module load {0}\n".format(module) - if params['cluster']=='ilifu': - contents = """#!/bin/bash{array}{exclude}{reservation} - #SBATCH --account={account} - #SBATCH --nodes={nodes} - #SBATCH --ntasks-per-node={tasks} - #SBATCH --cpus-per-task={cpus} - #SBATCH --mem={mem}GB - #SBATCH --job-name={runname}{name} - #SBATCH --distribution=plane={plane} - #SBATCH --output={LOG_DIR}/%x-{ID}.out - #SBATCH --error={LOG_DIR}/%x-{ID}.err - #SBATCH --partition={partition} - #SBATCH --time={time} - - export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK - {modules} - - {command}""" - elif params['cluster'] == 'galahad': - contents="""#!/bin/bash{array}{exclude}{reservation} - #SBATCH --nodes={nodes} - #SBATCH --threads=16 - #SBATCH --cpus-per-task=1 - #SBATCH --mem=1000GB - #SBATCH --job-name={runname}{name} - #SBATCH --output={LOG_DIR}/%x-{ID}.out - #SBATCH --error={LOG_DIR}/%x-{ID}.err - #SBATCH --partition={partition} - #SBATCH --time={time} - #SBATCH -w compute-0-100 - - {command} - """ + + contents = contents+"\nexport OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK\n{modules}\n\n{command}" + #insert arguments and remove whitespace contents = contents.format(**params).replace(" ","") @@ -526,9 +496,6 @@ def write_sbatch(script,args,mem,mpi_wrapper,nodes=1,tasks=16,name="job",runname config.close() logger.debug('Wrote sbatch file "{0}"'.format(sbatch)) - if params['cluster'] == 'galahad': - print("Galahad sbatch file content formated as:\n{0}".format(contents)) - logger.debug('Wrote sbatch file "{0}"'.format(sbatch)) def write_spw_master(filename,config,SPWs,precal_scripts,postcal_scripts,submit,dir='jobScripts',pad_length=5,dependencies='',timestamp='',slurm_kwargs={}): @@ -907,7 +874,7 @@ def srun(arg_dict,qos=True,time=10,mem=4): return call -def write_jobs(config, mpi_wrapper, mem, scripts=[], threadsafe=[], containers=[], num_precal_scripts=0, nodes=8, ntasks_per_node=4, plane=1, partition='Main', +def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], containers=[], num_precal_scripts=0, nodes=8, ntasks_per_node=4, plane=1, partition='Main', time='12:00:00', submit=False, name='', verbose=False, quiet=False, dependencies='', exclude='', account='b03-idia-ag', reservation='', modules=[], timestamp='', justrun=False): """Write a series of sbatch job files to calibrate a CASA MeasurementSet. @@ -971,10 +938,10 @@ def write_jobs(config, mpi_wrapper, mem, scripts=[], threadsafe=[], containers=[ #Use input SLURM configuration for threadsafe tasks, otherwise call srun with single node and single thread if threadsafe[i]: - write_sbatch(script,'--config {0}'.format(TMP_CONFIG),nodes=nodes,tasks=ntasks_per_node,mem=mem,plane=plane,exclude=exclude,mpi_wrapper=mpi_wrapper,container=containers[i],partition=partition, + write_sbatch(script,'--config {0}'.format(TMP_CONFIG),contents=contents,nodes=nodes,tasks=ntasks_per_node,mem=mem,plane=plane,exclude=exclude,mpi_wrapper=mpi_wrapper,container=containers[i],partition=partition, time=time,name=jobname,runname=name,SPWs=crosscal_kwargs['spw'],nspw=crosscal_kwargs['nspw'],account=account,reservation=reservation,modules=modules,justrun=justrun) else: - write_sbatch(script,'--config {0}'.format(TMP_CONFIG),nodes=1,tasks=1,mem=mem,plane=1,mpi_wrapper='srun',container=containers[i],partition=partition,time=time,name=jobname, + write_sbatch(script,'--config {0}'.format(TMP_CONFIG),contents=contents,nodes=1,tasks=1,mem=mem,plane=1,mpi_wrapper='srun',container=containers[i],partition=partition,time=time,name=jobname, runname=name,SPWs=crosscal_kwargs['spw'],nspw=crosscal_kwargs['nspw'],exclude=exclude,account=account,reservation=reservation,modules=modules,justrun=justrun) #Replace all .py with .sbatch @@ -1507,34 +1474,37 @@ def main(): else: ### Decide what to do when the cluster is not specified. pass # Set limits for current cluster configuration - TOTAL_NODES_LIMIT = DEFAULTS['TOTAL_NODES_LIMIT'] - CPUS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'] - NTASKS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'] - MEM_PER_NODE_GB_LIMIT = DEFAULTS['MEM_PER_NODE_GB_LIMIT'] - MEM_PER_NODE_GB_LIMIT_HIGHMEM = DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'] - ACCOUNTS = DEFAULTS['ACCOUNTS'] + TOTAL_NODES_LIMIT = DEFAULTS['TOTAL_NODES_LIMIT'.lower()] + CPUS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] + NTASKS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] + MEM_PER_NODE_GB_LIMIT = DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()] + MEM_PER_NODE_GB_LIMIT_HIGHMEM = DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'.lower()] + ACCOUNTS = DEFAULTS['ACCOUNTS'.lower()] # Set global values for paths and file names - LOG_DIR = DEFAULTS['LOG_DIR'] - CALIB_SCRIPTS_DIR = DEFAULTS['CALIB_SCRIPTS_DIR'] - AUX_SCRIPTS_DIR = DEFAULTS['AUX_SCRIPTS_DIR'] - SELFCAL_SCRIPTS_DIR = DEFAULTS['SELFCAL_SCRIPTS_DIR'] - CONFIG = DEFAULTS['CONFIG'] - TMP_CONFIG = DEFAULTS['TMP_CONFIG'] - MASTER_SCRIPT = DEFAULTS['MASTER_SCRIPT'] + LOG_DIR = DEFAULTS['LOG_DIR'.lower()] + CALIB_SCRIPTS_DIR = DEFAULTS['CALIB_SCRIPTS_DIR'.lower()] + AUX_SCRIPTS_DIR = DEFAULTS['AUX_SCRIPTS_DIR'.lower()] + SELFCAL_SCRIPTS_DIR = DEFAULTS['SELFCAL_SCRIPTS_DIR'.lower()] + CONFIG = DEFAULTS['CONFIG'.lower()] + TMP_CONFIG = DEFAULTS['TMP_CONFIG'.lower()] + MASTER_SCRIPT = DEFAULTS['MASTER_SCRIPT'.lower()] # Set global values for field, crosscal and SLURM arguments copied to config file - FIELDS_CONFIG_KEYS = DEFAULTS['FIELDS_CONFIG_KEYS'] - CROSSCAL_CONFIG_KEYS = DEFAULTS['CROSSCAL_CONFIG_KEYS'] - SELFCAL_CONFIG_KEYS = DEFAULTS['SELFCAL_CONFIG_KEYS'] - IMAGING_CONFIG_KEYS = DEFAULTS['IMAGING_CONFIG_KEYS'] - SLURM_CONFIG_STR_KEYS = DEFAULTS['SLURM_CONFIG_KEYS'] - SLURM_CONFIG_KEYS = DEFAULTS['SLURM_CONFIG_KEYS_BASE'] + SLURM_CONFIG_STR_KEYS - CONTAINER = DEFAULTS['CONTAINER'] - MPI_WRAPPER = DEFAULTS['MPI_WRAPPER'] - PRECAL_SCRIPTS = DEFAULTS['PRECAL_SCRIPTS'] - POSTCAL_SCRIPTS = DEFAULTS['POSTCAL_SCRIPTS'] - SCRIPTS = DEFAULTS['SCRIPTS'] + FIELDS_CONFIG_KEYS = DEFAULTS['FIELDS_CONFIG_KEYS'.lower()] + CROSSCAL_CONFIG_KEYS = DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()] + SELFCAL_CONFIG_KEYS = DEFAULTS['SELFCAL_CONFIG_KEYS'.lower()] + IMAGING_CONFIG_KEYS = DEFAULTS['IMAGING_CONFIG_KEYS'.lower()] + SLURM_CONFIG_STR_KEYS = DEFAULTS['SLURM_CONFIG_KEYS'.lower()] + SLURM_CONFIG_KEYS = DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()] + SLURM_CONFIG_STR_KEYS + CONTAINER = DEFAULTS['CONTAINER'.lower()] + MPI_WRAPPER = DEFAULTS['MPI_WRAPPER'.lower()] + PRECAL_SCRIPTS = DEFAULTS['PRECAL_SCRIPTS'.lower()] + POSTCAL_SCRIPTS = DEFAULTS['POSTCAL_SCRIPTS'.lower()] + SCRIPTS = DEFAULTS['SCRIPTS'.lower()] + + # Read in SBATCH file contents + contents = DEFAULTS['sbatch_file_base'] # Mutually exclusive arguments - display version, build config file or run pipeline if args.version: @@ -1545,7 +1515,7 @@ def main(): default_config(vars(args)) if args.run: kwargs = format_args(args.config,args.submit,args.quiet,args.dependencies,args.justrun) - write_jobs(args.config, mpi_wrapper=MPI_WRAPPER,**kwargs) + write_jobs(args.config, mpi_wrapper=MPI_WRAPPER, contents=contents,**kwargs) if __name__ == "__main__": main() From 310724d7f2983821b2534f34e135ef04e9af606e Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 4 Oct 2021 17:23:07 +0100 Subject: [PATCH 06/39] Renamed known_hpc.cfg --- processMeerKAT/{DEFAULTS.cfg => known_hpc.cfg} | 0 processMeerKAT/processMeerKAT.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename processMeerKAT/{DEFAULTS.cfg => known_hpc.cfg} (100%) diff --git a/processMeerKAT/DEFAULTS.cfg b/processMeerKAT/known_hpc.cfg similarity index 100% rename from processMeerKAT/DEFAULTS.cfg rename to processMeerKAT/known_hpc.cfg diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index c2d282e..29a993e 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -1458,7 +1458,7 @@ def main(): # Define defaults / limits for named HPC facilities THIS_PROG = __file__ SCRIPT_DIR = os.path.dirname(THIS_PROG) - DEFAULTS_CONFIG_PATH = "DEFAULTS.cfg" + DEFAULTS_CONFIG_PATH = "known_hpc.cfg" CONFIG_DEFAULTS,_ = config_parser.parse_config( "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) ) From 19747983b17d3cb1bf3289781d3a28b02eaa4dea Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 5 Oct 2021 08:47:23 +0100 Subject: [PATCH 07/39] Adding unknown HPC limits. --- processMeerKAT/known_hpc.cfg | 9 +++++ processMeerKAT/processMeerKAT.py | 63 ++++++++++++++++---------------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 619b79c..51c7b56 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -76,6 +76,15 @@ [ilifu] # [DEFAULTS] section defined at the top of this file represents the ilifu configuration. +[unknown] + # Differences to default: memory / node limits are functionally unlimited. + TOTAL_NODES_LIMIT = 5096 + CPUS_PER_NODE_LIMIT = 1024 + NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s + MEM_PER_NODE_GB_LIMIT = 5000 #237568 MB + MEM_PER_NODE_GB_LIMIT_HIGHMEM = 5000 #491520 MB + ACCOUNTS = [] + [galahad] # Specify differences to ilifu TOTAL_NODES_LIMIT = 1 ### update me! diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 29a993e..f7b0c45 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -244,9 +244,9 @@ def validate_args(args,config,parser=None): msg = "Only input an MS [-M --MS] during [-B --build] step. Otherwise input is ignored." raise_error(config, msg, parser) - if args['cluster'] not in KNOWN_CLUSTERS: + if args['cluster'] not in KNOWN_HPC: msg = "Cluster [--cluster] is not in {0}. You input {1}. Pipeline will rely entirely on the specified config. No upper limits will be set. HPC specific selections within your config which are not actually availble may cause pipeline runs to fail!" - logger.warning(msg.format(KNOWN_CLUSTERS, args['cluster'])) + logger.warning(msg.format(KNOWN_HPC, args['cluster'])) else: if args['ntasks_per_node'] > NTASKS_PER_NODE_LIMIT: @@ -1459,52 +1459,53 @@ def main(): THIS_PROG = __file__ SCRIPT_DIR = os.path.dirname(THIS_PROG) DEFAULTS_CONFIG_PATH = "known_hpc.cfg" - CONFIG_DEFAULTS,_ = config_parser.parse_config( + HPC_DEFAULTS,_ = config_parser.parse_config( "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) ) - KNOWN_CLUSTERS = DEFAULTS.keys() + KNOWN_HPC = HPC_DEFAULTS.keys() #Parse command-line arguments, and setup logger args = parse_args() setup_logger(args.config,args.verbose) # Select default source - if args.cluster in KNOWN_CLUSTERS: - DEFAULTS = CONFIG_DEFAULTS[args.cluster] - else: ### Decide what to do when the cluster is not specified. + if args.cluster in KNOWN_HPC: + HPC_DEFAULTS = HPC_DEFAULTS[args.cluster] + else: + HPC_DEFAULTS = HPC_DEFAULTS['unknown'] pass # Set limits for current cluster configuration - TOTAL_NODES_LIMIT = DEFAULTS['TOTAL_NODES_LIMIT'.lower()] - CPUS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] - NTASKS_PER_NODE_LIMIT = DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] - MEM_PER_NODE_GB_LIMIT = DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()] - MEM_PER_NODE_GB_LIMIT_HIGHMEM = DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'.lower()] + TOTAL_NODES_LIMIT = HPC_DEFAULTS['TOTAL_NODES_LIMIT'.lower()] + CPUS_PER_NODE_LIMIT = HPC_DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] + NTASKS_PER_NODE_LIMIT = HPC_DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] + MEM_PER_NODE_GB_LIMIT = HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()] + MEM_PER_NODE_GB_LIMIT_HIGHMEM = HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'.lower()] ACCOUNTS = DEFAULTS['ACCOUNTS'.lower()] # Set global values for paths and file names - LOG_DIR = DEFAULTS['LOG_DIR'.lower()] - CALIB_SCRIPTS_DIR = DEFAULTS['CALIB_SCRIPTS_DIR'.lower()] - AUX_SCRIPTS_DIR = DEFAULTS['AUX_SCRIPTS_DIR'.lower()] - SELFCAL_SCRIPTS_DIR = DEFAULTS['SELFCAL_SCRIPTS_DIR'.lower()] - CONFIG = DEFAULTS['CONFIG'.lower()] - TMP_CONFIG = DEFAULTS['TMP_CONFIG'.lower()] - MASTER_SCRIPT = DEFAULTS['MASTER_SCRIPT'.lower()] + LOG_DIR = HPC_DEFAULTS['LOG_DIR'.lower()] + CALIB_SCRIPTS_DIR = HPC_DEFAULTS['CALIB_SCRIPTS_DIR'.lower()] + AUX_SCRIPTS_DIR = HPC_DEFAULTS['AUX_SCRIPTS_DIR'.lower()] + SELFCAL_SCRIPTS_DIR = HPC_DEFAULTS['SELFCAL_SCRIPTS_DIR'.lower()] + CONFIG = HPC_DEFAULTS['CONFIG'.lower()] + TMP_CONFIG = HPC_DEFAULTS['TMP_CONFIG'.lower()] + MASTER_SCRIPT = HPC_DEFAULTS['MASTER_SCRIPT'.lower()] # Set global values for field, crosscal and SLURM arguments copied to config file - FIELDS_CONFIG_KEYS = DEFAULTS['FIELDS_CONFIG_KEYS'.lower()] - CROSSCAL_CONFIG_KEYS = DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()] - SELFCAL_CONFIG_KEYS = DEFAULTS['SELFCAL_CONFIG_KEYS'.lower()] - IMAGING_CONFIG_KEYS = DEFAULTS['IMAGING_CONFIG_KEYS'.lower()] - SLURM_CONFIG_STR_KEYS = DEFAULTS['SLURM_CONFIG_KEYS'.lower()] - SLURM_CONFIG_KEYS = DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()] + SLURM_CONFIG_STR_KEYS - CONTAINER = DEFAULTS['CONTAINER'.lower()] - MPI_WRAPPER = DEFAULTS['MPI_WRAPPER'.lower()] - PRECAL_SCRIPTS = DEFAULTS['PRECAL_SCRIPTS'.lower()] - POSTCAL_SCRIPTS = DEFAULTS['POSTCAL_SCRIPTS'.lower()] - SCRIPTS = DEFAULTS['SCRIPTS'.lower()] + FIELDS_CONFIG_KEYS = HPC_DEFAULTS['FIELDS_CONFIG_KEYS'.lower()] + CROSSCAL_CONFIG_KEYS = HPC_DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()] + SELFCAL_CONFIG_KEYS = HPC_DEFAULTS['SELFCAL_CONFIG_KEYS'.lower()] + IMAGING_CONFIG_KEYS = HPC_DEFAULTS['IMAGING_CONFIG_KEYS'.lower()] + SLURM_CONFIG_STR_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()] + SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()] + SLURM_CONFIG_STR_KEYS + CONTAINER = HPC_DEFAULTS['CONTAINER'.lower()] + MPI_WRAPPER = HPC_DEFAULTS['MPI_WRAPPER'.lower()] + PRECAL_SCRIPTS = HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()] + POSTCAL_SCRIPTS = HPC_DEFAULTS['POSTCAL_SCRIPTS'.lower()] + SCRIPTS = HPC_DEFAULTS['SCRIPTS'.lower()] # Read in SBATCH file contents - contents = DEFAULTS['sbatch_file_base'] + contents = HPC_DEFAULTS['sbatch_file_base'] # Mutually exclusive arguments - display version, build config file or run pipeline if args.version: From 6d0365946602404268034fa0bc377055f5637a19 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 5 Oct 2021 10:53:51 +0100 Subject: [PATCH 08/39] Fixing global access to defaults. --- processMeerKAT/processMeerKAT.py | 166 +++++++++++++------------------ 1 file changed, 68 insertions(+), 98 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index f7b0c45..dc46c02 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -65,12 +65,12 @@ def check_path(path,update=False): newpath = '../{0}'.format(path) elif os.path.exists('{0}/{1}'.format(SCRIPT_DIR,path)): newpath = '{0}/{1}'.format(SCRIPT_DIR,path) - elif os.path.exists('{0}/{1}/{2}'.format(SCRIPT_DIR,CALIB_SCRIPTS_DIR,path)): - newpath = '{0}/{1}/{2}'.format(SCRIPT_DIR,CALIB_SCRIPTS_DIR,path) - elif os.path.exists('{0}/{1}/{2}'.format(SCRIPT_DIR,AUX_SCRIPTS_DIR,path)): - newpath = '{0}/{1}/{2}'.format(SCRIPT_DIR,AUX_SCRIPTS_DIR,path) - elif os.path.exists('{0}/{1}/{2}'.format(SCRIPT_DIR,SELFCAL_SCRIPTS_DIR,path)): - newpath = '{0}/{1}/{2}'.format(SCRIPT_DIR,SELFCAL_SCRIPTS_DIR,path) + elif os.path.exists('{0}/{1}/{2}'.format(SCRIPT_DIR, HPC_DEFAULTS['CALIB_SCRIPTS_DIR'.lower()], path)): + newpath = '{0}/{1}/{2}'.format(SCRIPT_DIR, HPC_DEFAULTS['CALIB_SCRIPTS_DIR'.lower()], path) + elif os.path.exists('{0}/{1}/{2}'.format(SCRIPT_DIR, HPC_DEFAULTS['AUX_SCRIPTS_DIR'.lower()], path)): + newpath = '{0}/{1}/{2}'.format(SCRIPT_DIR, HPC_DEFAULTS['AUX_SCRIPTS_DIR'.lower()], path) + elif os.path.exists('{0}/{1}/{2}'.format(SCRIPT_DIR, HPC_DEFAULTS['SELFCAL_SCRIPTS_DIR'.lower()], path)): + newpath = '{0}/{1}/{2}'.format(SCRIPT_DIR, HPC_DEFAULTS['SELFCAL_SCRIPTS_DIR'.lower()], path) elif os.path.exists(check_bash_path(path)): newpath = check_bash_path(path) else: @@ -134,25 +134,25 @@ def parse_scripts(val): parser.add_argument("--cluster",metavar='name', required=False, type=str, default="ilifu", help="Name of cluster being used if not ilifu default slurm limits are removed [default: ilifu].") parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") - parser.add_argument("-C","--config",metavar="path", default=CONFIG, required=False, type=str, help="Relative (not absolute) path to config file.") + parser.add_argument("-C","--config",metavar="path", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") parser.add_argument("-N","--nodes",metavar="num", required=False, type=int, default=1, - help="Use this number of nodes [default: 1; max: {0}].".format(TOTAL_NODES_LIMIT)) + help="Use this number of nodes [default: 1; max: {0}].".format(HPC_DEFAULTS['TOTAL_NODES_LIMIT'.lower()])) parser.add_argument("-t","--ntasks-per-node", metavar="num", required=False, type=int, default=8, - help="Use this number of tasks (per node) [default: 16; max: {0}].".format(NTASKS_PER_NODE_LIMIT)) + help="Use this number of tasks (per node) [default: 16; max: {0}].".format(HPC_DEFAULTS['NTASKS_PER_NODE_LIMIT'.lower()])) parser.add_argument("-D","--plane", metavar="num", required=False, type=int, default=1, help="Distribute tasks of this block size before moving onto next node [default: 1; max: ntasks-per-node].") - parser.add_argument("-m","--mem", metavar="num", required=False, type=int, default=MEM_PER_NODE_GB_LIMIT, - help="Use this many GB of memory (per node) for threadsafe scripts [default: {0}; max: {0}].".format(MEM_PER_NODE_GB_LIMIT)) + parser.add_argument("-m","--mem", metavar="num", required=False, type=int, default=HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()], + help="Use this many GB of memory (per node) for threadsafe scripts [default: {0}; max: {0}].".format(HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()])) parser.add_argument("-p","--partition", metavar="name", required=False, type=str, default="Main", help="SLURM partition to use [default: 'Main'].") parser.add_argument("-T","--time", metavar="time", required=False, type=str, default="12:00:00", help="Time limit to use for all jobs, in the form d-hh:mm:ss [default: '12:00:00'].") - parser.add_argument("-S","--scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=SCRIPTS, + parser.add_argument("-S","--scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['SCRIPTS'.lower()], help="Run pipeline with these scripts, in this order, using these containers (3rd value - empty string to default to [-c --container]). Is it threadsafe (2nd value)?") - parser.add_argument("-b","--precal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=PRECAL_SCRIPTS, help="Same as [-S --scripts], but run before calibration.") - parser.add_argument("-a","--postcal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=POSTCAL_SCRIPTS, help="Same as [-S --scripts], but run after calibration.") + parser.add_argument("-b","--precal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()], help="Same as [-S --scripts], but run before calibration.") + parser.add_argument("-a","--postcal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['POSTCAL_SCRIPTS'], help="Same as [-S --scripts], but run after calibration.") parser.add_argument("--modules", nargs='*', metavar='module', required=False, default=['openmpi/2.1.1'], help="Load these modules within each sbatch script.") - parser.add_argument("-w","--mpi_wrapper", metavar="path", required=False, type=str, default=MPI_WRAPPER, - help="Use this mpi wrapper when calling threadsafe scripts [default: '{0}'].".format(MPI_WRAPPER)) - parser.add_argument("-c","--container", metavar="path", required=False, type=str, default=CONTAINER, help="Use this container when calling scripts [default: '{0}'].".format(CONTAINER)) + parser.add_argument("-w","--mpi_wrapper", metavar="path", required=False, type=str, default=HPC_DEFAULTS['MPI_WRAPPER'.lower()], + help="Use this mpi wrapper when calling threadsafe scripts [default: '{0}'].".format(HPC_DEFAULTS['MPI_WRAPPER'.lower()])) + parser.add_argument("-c","--container", metavar="path", required=False, type=str, default=CONTAINER, help="Use this container when calling scripts [default: '{0}'].".format(HPC_DEFAULTS['CONTAINER'.lower()])) parser.add_argument("-n","--name", metavar="unique", required=False, type=str, default='', help="Unique name to give this pipeline run (e.g. 'run1_'), appended to the start of all job names. [default: ''].") parser.add_argument("-d","--dependencies", metavar="list", required=False, type=str, default='', help="Comma-separated list (without spaces) of SLURM job dependencies (only used when nspw=1). [default: ''].") parser.add_argument("-e","--exclude", metavar="nodes", required=False, type=str, default='', help="SLURM worker nodes to exclude [default: ''].") @@ -188,12 +188,12 @@ def parse_scripts(val): parser.error("Input config file '{0}' not found. Please set [-C --config] or write a new one with [-B --build].".format(args.config)) #if user inputs a list a scripts, remove the default list - if len(args.scripts) > len(SCRIPTS): - [args.scripts.pop(0) for i in range(len(SCRIPTS))] - if len(args.precal_scripts) > len(PRECAL_SCRIPTS): - [args.precal_scripts.pop(0) for i in range(len(PRECAL_SCRIPTS))] - if len(args.postcal_scripts) > len(POSTCAL_SCRIPTS): - [args.postcal_scripts.pop(0) for i in range(len(POSTCAL_SCRIPTS))] + if len(args.scripts) > len(HPC_DEFAULTS['SCRIPTS'.lower()]): + [args.scripts.pop(0) for i in range(len(HPC_DEFAULTS['SCRIPTS'.lower()]))] + if len(args.precal_scripts) > len(HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()]): + [args.precal_scripts.pop(0) for i in range(len(HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()]))] + if len(args.postcal_scripts) > len(HPC_DEFAULTS['POSTCAL_SCRIPTS'.lower()]): + [args.postcal_scripts.pop(0) for i in range(len(HPC_DEFAULTS['POSTCAL_SCRIPTS'.lower()]))] #validate arguments before returning them validate_args(vars(args),args.config,parser=parser) @@ -244,32 +244,32 @@ def validate_args(args,config,parser=None): msg = "Only input an MS [-M --MS] during [-B --build] step. Otherwise input is ignored." raise_error(config, msg, parser) - if args['cluster'] not in KNOWN_HPC: + if args['cluster'] not in HPC_DEFAULTS.keys(): msg = "Cluster [--cluster] is not in {0}. You input {1}. Pipeline will rely entirely on the specified config. No upper limits will be set. HPC specific selections within your config which are not actually availble may cause pipeline runs to fail!" - logger.warning(msg.format(KNOWN_HPC, args['cluster'])) + logger.warning(msg.format(HPC_DEFAULTS.keys(), args['cluster'])) else: - if args['ntasks_per_node'] > NTASKS_PER_NODE_LIMIT: - msg = "The number of tasks per node [-t --ntasks-per-node] must not exceed {0}. You input {1}.".format(NTASKS_PER_NODE_LIMIT,args['ntasks_per_node']) + if args['ntasks_per_node'] > HPC_DEFAULTS['NTASKS_PER_NODE_LIMIT'.lower()]: + msg = "The number of tasks per node [-t --ntasks-per-node] must not exceed {0}. You input {1}.".format(HPC_DEFAULTS['NTASKS_PER_NODE_LIMIT'.lower()],args['ntasks_per_node']) raise_error(config, msg, parser) - if args['nodes'] > TOTAL_NODES_LIMIT: - msg = "The number of nodes [-N --nodes] per node must not exceed {0}. You input {1}.".format(TOTAL_NODES_LIMIT,args['nodes']) + if args['nodes'] > HPC_DEFAULTS['TOTAL_NODES_LIMIT'.lower()]: + msg = "The number of nodes [-N --nodes] per node must not exceed {0}. You input {1}.".format(HPC_DEFAULTS['TOTAL_NODES_LIMIT'.lower()],args['nodes']) raise_error(config, msg, parser) - if args['mem'] > MEM_PER_NODE_GB_LIMIT: + if args['mem'] > HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()]: if args['partition'] != 'HighMem': - msg = "The memory per node [-m --mem] must not exceed {0} (GB). You input {1} (GB).".format(MEM_PER_NODE_GB_LIMIT,args['mem']) + msg = "The memory per node [-m --mem] must not exceed {0} (GB). You input {1} (GB).".format(HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()],args['mem']) raise_error(config, msg, parser) - elif args['mem'] > MEM_PER_NODE_GB_LIMIT_HIGHMEM: - msg = "The memory per node [-m --mem] must not exceed {0} (GB) when using 'HighMem' partition. You input {1} (GB).".format(MEM_PER_NODE_GB_LIMIT_HIGHMEM,args['mem']) + elif args['mem'] > HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'.lower()]: + msg = "The memory per node [-m --mem] must not exceed {0} (GB) when using 'HighMem' partition. You input {1} (GB).".format(HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'.lower()],args['mem']) raise_error(config, msg, parser) if args['plane'] > args['ntasks_per_node']: msg = "The value of [-P --plane] cannot be greater than the tasks per node [-t --ntasks-per-node] ({0}). You input {1}.".format(args['ntasks_per_node'],args['plane']) raise_error(config, msg, parser) - if args['account'] not in ACCOUNTS: + if args['account'] not in HPC_DEFAULTS['ACCOUNTS'.lower()]: from platform import node if 'slurm-login' in node() or 'slwrk' in node() or 'compute' in node(): accounts=os.popen("for f in $(sacctmgr show user $USER --noheader cluster=ilifu-slurm20 -s format=account%30); do echo -n $f,; done").read()[:-1].split(',') @@ -335,7 +335,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False #Store parameters passed into this function as dictionary, and add to it params = locals() - params['LOG_DIR'] = LOG_DIR + params['LOG_DIR'] = HPC_DEFAULTS['LOG_DIR'.lower()] params['job'] = '${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}' if arrayJob else '${SLURM_JOB_ID}' params['job'] = '${SLURM_JOB_NAME}-' + params['job'] params['casa_call'] = '' @@ -420,31 +420,31 @@ def write_sbatch(script,args,mem,mpi_wrapper,contents,nodes=1,tasks=16,name="job justrun : bool, optionall Just run the pipeline without rebuilding each job script (if it exists).""" - if not os.path.exists(LOG_DIR): - os.mkdir(LOG_DIR) + if not os.path.exists(HPC_DEFAULTS['LOG_DIR'.lower()]): + os.mkdir(HPC_DEFAULTS['LOG_DIR'.lower()]) #Store parameters passed into this function as dictionary, and add to it params = locals() - params['LOG_DIR'] = LOG_DIR + params['LOG_DIR'] = HPC_DEFAULTS['LOG_DIR'.lower()] #Use multiple CPUs for tclean and paratition scripts params['cpus'] = 1 if 'tclean' in script or 'selfcal' in script or 'partition' in script or 'image' in script: - params['cpus'] = int(CPUS_PER_NODE_LIMIT/tasks) + params['cpus'] = int(HPC_DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()]/tasks) #hard-code for 2/4 polarisations if 'partition' in script: - dopol = config_parser.get_key(TMP_CONFIG, 'run', 'dopol') - if dopol and 4*tasks < CPUS_PER_NODE_LIMIT: + dopol = config_parser.get_key(HPC_DEFAULTS['TMP_CONFIG'.lower()], 'run', 'dopol') + if dopol and 4*tasks < HPC_DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()]: params['cpus'] = 4 elif not dopol and params['cpus'] > 2: params['cpus'] = 2 #If requesting all CPUs, user may as well use all memory - if params['cpus'] * tasks == CPUS_PER_NODE_LIMIT: + if params['cpus'] * tasks == HPC_DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()]: if params['partition'] == 'HighMem': - params['mem'] = MEM_PER_NODE_GB_LIMIT_HIGHMEM + params['mem'] = HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'.lower()] else: - params['mem'] = MEM_PER_NODE_GB_LIMIT + params['mem'] = HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()] #Use xvfb for plotting scripts plot = ('plot' in script) @@ -701,8 +701,8 @@ def write_master(filename,config,scripts=[],submit=False,dir='jobScripts',pad_le #Copy config file to TMP_CONFIG and inform user if verbose: - master.write("\necho Copying \'{0}\' to \'{1}\', and using this to run pipeline.\n".format(config,TMP_CONFIG)) - master.write('cp {0} {1}\n'.format(config, TMP_CONFIG)) + master.write("\necho Copying \'{0}\' to \'{1}\', and using this to run pipeline.\n".format(config,HPC_DEFAULTS['TMP_CONFIG'.lower()])) + master.write('cp {0} {1}\n'.format(config, HPC_DEFAULTS['TMP_CONFIG'.lower()])) #Hack to perform correct number of selfcal loops if config_parser.has_section(config,'selfcal') and 'selfcal_part1.sbatch' in scripts and 'selfcal_part2.sbatch' in scripts: @@ -800,6 +800,7 @@ def write_all_bash_jobs_scripts(master,extn,IDs,dir='jobScripts',echo=True,prefi write_bash_job_script(master, killScript, extn, 'echo scancel ${0}'.format(IDs), 'kill all the jobs', dir=dir, echo=echo) do = """echo sacct -j ${0} --units=G -o "JobID%-15,JobName%-{1},Partition,Elapsed,NNodes%6,NTasks%6,NCPUS%5,MaxDiskRead,MaxDiskWrite,NodeList%20,TotalCPU,CPUTime,MaxRSS,State,ExitCode" \$@ """.format(IDs,15+pad_length) write_bash_job_script(master, summaryScript, extn, do, 'view the progress', dir=dir, echo=echo) + LOG_DIR = HPC_DEFAULTS['LOG_DIR'.lower()] do = """echo "for ID in {$%s,}; do files=\$(ls %s/*\$ID* 2>/dev/null | wc -l); if [ \$((files)) != 0 ]; then ls %s/*\$ID*; cat %s/*\$ID* | grep -i 'severe\|error' | grep -vi 'mpi\|The selected table has zero rows\|MeasTable::dUTC(Double)'; else echo %s/*\$ID* logs don\\'t exist \(yet\); fi; done" """ % (IDs,LOG_DIR,LOG_DIR,LOG_DIR,LOG_DIR) write_bash_job_script(master, errorScript, extn, do, 'find errors \(after pipeline has run\)', dir=dir, echo=echo) do = """echo "for ID in {$%s,}; do files=\$(ls %s/*\$ID* 2>/dev/null | wc -l); if [ \$((files)) != 0 ]; then logs=\$(ls %s/*\$ID* | sort -V); ls -f \$logs; cat \$(ls -tU \$logs) | grep INFO | head -n 1 | cut -d 'I' -f1; cat \$(ls -tr \$logs) | grep INFO | tail -n 1 | cut -d 'I' -f1; else echo %s/*\$ID* logs don\\'t exist \(yet\); fi; done" """ % (IDs,LOG_DIR,LOG_DIR,LOG_DIR) @@ -929,7 +930,7 @@ def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], co Just run the pipeline without rebuilding each job script (if it exists).""" kwargs = locals() - crosscal_kwargs = get_config_kwargs(config, 'crosscal', CROSSCAL_CONFIG_KEYS) + crosscal_kwargs = get_config_kwargs(config, 'crosscal', HPC_DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()]) pad_length = len(name) #Write sbatch file for each input python script @@ -938,10 +939,10 @@ def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], co #Use input SLURM configuration for threadsafe tasks, otherwise call srun with single node and single thread if threadsafe[i]: - write_sbatch(script,'--config {0}'.format(TMP_CONFIG),contents=contents,nodes=nodes,tasks=ntasks_per_node,mem=mem,plane=plane,exclude=exclude,mpi_wrapper=mpi_wrapper,container=containers[i],partition=partition, + write_sbatch(script,'--config {0}'.format(HPC_DEFAULTS['TMP_CONFIG'.lower()]),contents=contents,nodes=nodes,tasks=ntasks_per_node,mem=mem,plane=plane,exclude=exclude,mpi_wrapper=mpi_wrapper,container=containers[i],partition=partition, time=time,name=jobname,runname=name,SPWs=crosscal_kwargs['spw'],nspw=crosscal_kwargs['nspw'],account=account,reservation=reservation,modules=modules,justrun=justrun) else: - write_sbatch(script,'--config {0}'.format(TMP_CONFIG),contents=contents,nodes=1,tasks=1,mem=mem,plane=1,mpi_wrapper='srun',container=containers[i],partition=partition,time=time,name=jobname, + write_sbatch(script,'--config {0}'.format(HPC_DEFAULTS['TMP_CONFIG']),contents=contents,nodes=1,tasks=1,mem=mem,plane=1,mpi_wrapper='srun',container=containers[i],partition=partition,time=time,name=jobname, runname=name,SPWs=crosscal_kwargs['spw'],nspw=crosscal_kwargs['nspw'],exclude=exclude,account=account,reservation=reservation,modules=modules,justrun=justrun) #Replace all .py with .sbatch @@ -952,10 +953,10 @@ def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], co if crosscal_kwargs['nspw'] > 1: #Build master master script, calling each of the separate SPWs at once, precal scripts before this, and postcal scripts after this - write_spw_master(MASTER_SCRIPT,config,SPWs=crosscal_kwargs['spw'],precal_scripts=precal_scripts,postcal_scripts=postcal_scripts,submit=submit,pad_length=pad_length,dependencies=dependencies,timestamp=timestamp,slurm_kwargs=kwargs) + write_spw_master(HPC_DEFAULTS['MASTER_SCRIPT'.lower()],config,SPWs=crosscal_kwargs['spw'],precal_scripts=precal_scripts,postcal_scripts=postcal_scripts,submit=submit,pad_length=pad_length,dependencies=dependencies,timestamp=timestamp,slurm_kwargs=kwargs) else: #Build master pipeline submission script - write_master(MASTER_SCRIPT,config,scripts=scripts,submit=submit,pad_length=pad_length,verbose=verbose,echo=echo,dependencies=dependencies,slurm_kwargs=kwargs) + write_master(HPC_DEFAULTS['MASTER_SCRIPT'.lower()],config,scripts=scripts,submit=submit,pad_length=pad_length,verbose=verbose,echo=echo,dependencies=dependencies,slurm_kwargs=kwargs) def default_config(arg_dict): @@ -971,11 +972,11 @@ def default_config(arg_dict): MS = arg_dict['MS'] #Copy default config to current location - copyfile('{0}/{1}'.format(SCRIPT_DIR,CONFIG),filename) + copyfile('{0}/{1}'.format(SCRIPT_DIR,HPC_DEFAULTS['CONFIG'.lower()]),filename) #Add SLURM CL arguments to config file under section [slurm] - slurm_dict = get_slurm_dict(arg_dict,SLURM_CONFIG_KEYS) - for key in SLURM_CONFIG_STR_KEYS: + slurm_dict = get_slurm_dict(arg_dict,HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()]) + for key in HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'.lower()]: if key in slurm_dict.keys(): slurm_dict[key] = "'{0}'".format(slurm_dict[key]) #Overwrite CL parameters in config under section [slurm] @@ -1113,20 +1114,20 @@ def format_args(config,submit,quiet,dependencies,justrun): Keyword arguments extracted from [slurm] section of config file, to be passed into write_jobs() function.""" #Ensure all keys exist in these sections - kwargs = get_config_kwargs(config,'slurm',SLURM_CONFIG_KEYS) + kwargs = get_config_kwargs(config,'slurm',HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()]) data_kwargs = get_config_kwargs(config,'data',['vis']) - get_config_kwargs(config, 'fields', FIELDS_CONFIG_KEYS) - crosscal_kwargs = get_config_kwargs(config, 'crosscal', CROSSCAL_CONFIG_KEYS) + get_config_kwargs(config, 'fields', HPC_DEFAULTS['FIELDS_CONFIG_KEYS'.lower()]) + crosscal_kwargs = get_config_kwargs(config, 'crosscal', HPC_DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()]) #Check selfcal params if config_parser.has_section(config,'selfcal'): - selfcal_kwargs = get_config_kwargs(config, 'selfcal', SELFCAL_CONFIG_KEYS) + selfcal_kwargs = get_config_kwargs(config, 'selfcal', HPC_DEFAULTS['SELFCAL_CONFIG_KEYS'.lower()]) bookkeeping.get_selfcal_params() if selfcal_kwargs['loop'] > 0: logger.warning("Starting with loop={0}, which is only valid if previous loops were successfully run in this directory.".format(selfcal_kwargs['loop'])) if config_parser.has_section(config,'image'): - imaging_kwargs = get_config_kwargs(config, 'image', IMAGING_CONFIG_KEYS) + imaging_kwargs = get_config_kwargs(config, 'image', HPC_DEFAULTS['IMAGING_CONFIG_KEYS'.lower()]) #Force submit=True if user has requested it during [-R --run] if submit: @@ -1159,7 +1160,7 @@ def format_args(config,submit,quiet,dependencies,justrun): config_parser.overwrite_config(config, conf_dict={'scripts' : scripts}, conf_sec='slurm') config_parser.overwrite_config(config, conf_dict={'precal_scripts' : []}, conf_sec='slurm') config_parser.overwrite_config(config, conf_dict={'postcal_scripts' : []}, conf_sec='slurm') - kwargs = get_config_kwargs(config,'slurm',SLURM_CONFIG_KEYS) + kwargs = get_config_kwargs(config,'slurm', HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()]) else: scripts = kwargs['scripts'] else: @@ -1195,7 +1196,7 @@ def format_args(config,submit,quiet,dependencies,justrun): kwargs['threadsafe'][kwargs['scripts'].index(threadsafe_script)] = True #Only reduce the memory footprint if we're not using all CPUs on each node - if kwargs['ntasks_per_node'] < NTASKS_PER_NODE_LIMIT and nspw > 1: + if kwargs['ntasks_per_node'] < HPC_DEFAULTS['NTASKS_PER_NODE_LIMIT'.lower()] and nspw > 1: mem = int(mem // (nspw/2)) dopol = config_parser.get_key(config, 'run', 'dopol') @@ -1237,8 +1238,8 @@ def format_args(config,submit,quiet,dependencies,justrun): #sys.exit(1) #If everything up until here has passed, we can copy config file to TMP_CONFIG (in case user runs sbatch manually) and inform user - logger.debug("Copying '{0}' to '{1}', and using this to run pipeline.".format(config,TMP_CONFIG)) - copyfile(config, TMP_CONFIG) + logger.debug("Copying '{0}' to '{1}', and using this to run pipeline.".format(config, HPC_DEFAULTS['TMP_CONFIG'.lower()])) + copyfile(config, HPC_DEFAULTS['TMP_CONFIG'.lower()]) if not quiet: logger.warning("Changing [slurm] section in your config will have no effect unless you [-R --run] again.") @@ -1454,6 +1455,8 @@ def setup_logger(config,verbose=False): logger.setLevel(loglevel) def main(): + # Define global variables (module level) + global THIS_PROG, SCRIPT_DIR, HPC_DEFAULTS # Define defaults / limits for named HPC facilities THIS_PROG = __file__ @@ -1462,50 +1465,17 @@ def main(): HPC_DEFAULTS,_ = config_parser.parse_config( "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) ) - KNOWN_HPC = HPC_DEFAULTS.keys() #Parse command-line arguments, and setup logger args = parse_args() setup_logger(args.config,args.verbose) # Select default source + KNOWN_HPC = HPC_DEFAULTS.keys() if args.cluster in KNOWN_HPC: HPC_DEFAULTS = HPC_DEFAULTS[args.cluster] else: HPC_DEFAULTS = HPC_DEFAULTS['unknown'] - pass - # Set limits for current cluster configuration - TOTAL_NODES_LIMIT = HPC_DEFAULTS['TOTAL_NODES_LIMIT'.lower()] - CPUS_PER_NODE_LIMIT = HPC_DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] - NTASKS_PER_NODE_LIMIT = HPC_DEFAULTS['CPUS_PER_NODE_LIMIT'.lower()] - MEM_PER_NODE_GB_LIMIT = HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()] - MEM_PER_NODE_GB_LIMIT_HIGHMEM = HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT_HIGHMEM'.lower()] - ACCOUNTS = DEFAULTS['ACCOUNTS'.lower()] - - # Set global values for paths and file names - LOG_DIR = HPC_DEFAULTS['LOG_DIR'.lower()] - CALIB_SCRIPTS_DIR = HPC_DEFAULTS['CALIB_SCRIPTS_DIR'.lower()] - AUX_SCRIPTS_DIR = HPC_DEFAULTS['AUX_SCRIPTS_DIR'.lower()] - SELFCAL_SCRIPTS_DIR = HPC_DEFAULTS['SELFCAL_SCRIPTS_DIR'.lower()] - CONFIG = HPC_DEFAULTS['CONFIG'.lower()] - TMP_CONFIG = HPC_DEFAULTS['TMP_CONFIG'.lower()] - MASTER_SCRIPT = HPC_DEFAULTS['MASTER_SCRIPT'.lower()] - - # Set global values for field, crosscal and SLURM arguments copied to config file - FIELDS_CONFIG_KEYS = HPC_DEFAULTS['FIELDS_CONFIG_KEYS'.lower()] - CROSSCAL_CONFIG_KEYS = HPC_DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()] - SELFCAL_CONFIG_KEYS = HPC_DEFAULTS['SELFCAL_CONFIG_KEYS'.lower()] - IMAGING_CONFIG_KEYS = HPC_DEFAULTS['IMAGING_CONFIG_KEYS'.lower()] - SLURM_CONFIG_STR_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()] - SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()] + SLURM_CONFIG_STR_KEYS - CONTAINER = HPC_DEFAULTS['CONTAINER'.lower()] - MPI_WRAPPER = HPC_DEFAULTS['MPI_WRAPPER'.lower()] - PRECAL_SCRIPTS = HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()] - POSTCAL_SCRIPTS = HPC_DEFAULTS['POSTCAL_SCRIPTS'.lower()] - SCRIPTS = HPC_DEFAULTS['SCRIPTS'.lower()] - - # Read in SBATCH file contents - contents = HPC_DEFAULTS['sbatch_file_base'] # Mutually exclusive arguments - display version, build config file or run pipeline if args.version: @@ -1516,7 +1486,7 @@ def main(): default_config(vars(args)) if args.run: kwargs = format_args(args.config,args.submit,args.quiet,args.dependencies,args.justrun) - write_jobs(args.config, mpi_wrapper=MPI_WRAPPER, contents=contents,**kwargs) + write_jobs(args.config, mpi_wrapper=MPI_WRAPPER, contents=HPC_DEFAULTS['sbatch_file_base'], **kwargs) if __name__ == "__main__": main() From 4030cba91d31725a26ed86d41c4e2138a949ded3 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 5 Oct 2021 12:43:00 +0100 Subject: [PATCH 09/39] Intermittent Argparse to set default HPC values. Including bug fixes around default calls. --- processMeerKAT/known_hpc.cfg | 2 +- processMeerKAT/processMeerKAT.py | 42 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 51c7b56..5dcf1ed 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -92,7 +92,7 @@ NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s MEM_PER_NODE_GB_LIMIT = 1000 #237568 MB ### update me!!! MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1300 #491520 MB ### update me!!! - ACCOUNTS = [] ### update me!!! + ACCOUNTS = ['b03-idia-ag'] ### update me!!! CONTAINER = '/share/nas/mbowles/mightee/casa-stable.simg' MPI_WRAPPER = %(CONTAINER)s ### update me!!! # Fairly certain this shouldnt work, but it might. I.e. should probably still reference mpirun somehow. # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index dc46c02..b68a68b 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -132,7 +132,13 @@ def parse_scripts(val): parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) - parser.add_argument("--cluster",metavar='name', required=False, type=str, default="ilifu", help="Name of cluster being used if not ilifu default slurm limits are removed [default: ilifu].") + parser.add_argument("--hpc",metavar='name', required=False, type=str, default="ilifu", help="Name of hpc facility being used if not known to the config (processMeerKAT/known_hpc.cfg) slurm limits are functionally removed [default: ilifu].") + # Read in default values according to --cluster parameter + args, unknown = parser.parse_known_args() + global HPC_DEFAULTS, HPC + HPC = args.hpc if args.hpc in HPC_DEFAULTS.keys() else "unknown" + HPC_DEFAULTS = HPC_DEFAULTS[HPC] + parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") parser.add_argument("-C","--config",metavar="path", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") parser.add_argument("-N","--nodes",metavar="num", required=False, type=int, default=1, @@ -148,11 +154,11 @@ def parse_scripts(val): parser.add_argument("-S","--scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['SCRIPTS'.lower()], help="Run pipeline with these scripts, in this order, using these containers (3rd value - empty string to default to [-c --container]). Is it threadsafe (2nd value)?") parser.add_argument("-b","--precal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()], help="Same as [-S --scripts], but run before calibration.") - parser.add_argument("-a","--postcal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['POSTCAL_SCRIPTS'], help="Same as [-S --scripts], but run after calibration.") + parser.add_argument("-a","--postcal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['POSTCAL_SCRIPTS'.lower()], help="Same as [-S --scripts], but run after calibration.") parser.add_argument("--modules", nargs='*', metavar='module', required=False, default=['openmpi/2.1.1'], help="Load these modules within each sbatch script.") parser.add_argument("-w","--mpi_wrapper", metavar="path", required=False, type=str, default=HPC_DEFAULTS['MPI_WRAPPER'.lower()], help="Use this mpi wrapper when calling threadsafe scripts [default: '{0}'].".format(HPC_DEFAULTS['MPI_WRAPPER'.lower()])) - parser.add_argument("-c","--container", metavar="path", required=False, type=str, default=CONTAINER, help="Use this container when calling scripts [default: '{0}'].".format(HPC_DEFAULTS['CONTAINER'.lower()])) + parser.add_argument("-c","--container", metavar="path", required=False, type=str, default=HPC_DEFAULTS['CONTAINER'.lower()], help="Use this container when calling scripts [default: '{0}'].".format(HPC_DEFAULTS['CONTAINER'.lower()])) parser.add_argument("-n","--name", metavar="unique", required=False, type=str, default='', help="Unique name to give this pipeline run (e.g. 'run1_'), appended to the start of all job names. [default: ''].") parser.add_argument("-d","--dependencies", metavar="list", required=False, type=str, default='', help="Comma-separated list (without spaces) of SLURM job dependencies (only used when nspw=1). [default: ''].") parser.add_argument("-e","--exclude", metavar="nodes", required=False, type=str, default='', help="SLURM worker nodes to exclude [default: ''].") @@ -244,9 +250,9 @@ def validate_args(args,config,parser=None): msg = "Only input an MS [-M --MS] during [-B --build] step. Otherwise input is ignored." raise_error(config, msg, parser) - if args['cluster'] not in HPC_DEFAULTS.keys(): - msg = "Cluster [--cluster] is not in {0}. You input {1}. Pipeline will rely entirely on the specified config. No upper limits will be set. HPC specific selections within your config which are not actually availble may cause pipeline runs to fail!" - logger.warning(msg.format(HPC_DEFAULTS.keys(), args['cluster'])) + if HPC=="unknown": + msg = "HPC facility [--hpc] is not in 'known_hpc.cfg', reverting to 'unknown' HPC. You input {0}. Pipeline will rely entirely on the specified arguemnts. No upper limits will be set. HPC specific selections within your config may cause pipeline runs to fail!" + logger.warning(msg.format(args['hpc'])) else: if args['ntasks_per_node'] > HPC_DEFAULTS['NTASKS_PER_NODE_LIMIT'.lower()]: @@ -942,7 +948,7 @@ def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], co write_sbatch(script,'--config {0}'.format(HPC_DEFAULTS['TMP_CONFIG'.lower()]),contents=contents,nodes=nodes,tasks=ntasks_per_node,mem=mem,plane=plane,exclude=exclude,mpi_wrapper=mpi_wrapper,container=containers[i],partition=partition, time=time,name=jobname,runname=name,SPWs=crosscal_kwargs['spw'],nspw=crosscal_kwargs['nspw'],account=account,reservation=reservation,modules=modules,justrun=justrun) else: - write_sbatch(script,'--config {0}'.format(HPC_DEFAULTS['TMP_CONFIG']),contents=contents,nodes=1,tasks=1,mem=mem,plane=1,mpi_wrapper='srun',container=containers[i],partition=partition,time=time,name=jobname, + write_sbatch(script,'--config {0}'.format(HPC_DEFAULTS['TMP_CONFIG'.lower()]),contents=contents,nodes=1,tasks=1,mem=mem,plane=1,mpi_wrapper='srun',container=containers[i],partition=partition,time=time,name=jobname, runname=name,SPWs=crosscal_kwargs['spw'],nspw=crosscal_kwargs['nspw'],exclude=exclude,account=account,reservation=reservation,modules=modules,justrun=justrun) #Replace all .py with .sbatch @@ -975,7 +981,8 @@ def default_config(arg_dict): copyfile('{0}/{1}'.format(SCRIPT_DIR,HPC_DEFAULTS['CONFIG'.lower()]),filename) #Add SLURM CL arguments to config file under section [slurm] - slurm_dict = get_slurm_dict(arg_dict,HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()]) + SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()] + HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'.lower()] + slurm_dict = get_slurm_dict(arg_dict, SLURM_CONFIG_KEYS) for key in HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'.lower()]: if key in slurm_dict.keys(): slurm_dict[key] = "'{0}'".format(slurm_dict[key]) @@ -1114,7 +1121,8 @@ def format_args(config,submit,quiet,dependencies,justrun): Keyword arguments extracted from [slurm] section of config file, to be passed into write_jobs() function.""" #Ensure all keys exist in these sections - kwargs = get_config_kwargs(config,'slurm',HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()]) + SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE']+HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'] + kwargs = get_config_kwargs(config,'slurm',SLURM_CONFIG_KEYS) data_kwargs = get_config_kwargs(config,'data',['vis']) get_config_kwargs(config, 'fields', HPC_DEFAULTS['FIELDS_CONFIG_KEYS'.lower()]) crosscal_kwargs = get_config_kwargs(config, 'crosscal', HPC_DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()]) @@ -1160,7 +1168,8 @@ def format_args(config,submit,quiet,dependencies,justrun): config_parser.overwrite_config(config, conf_dict={'scripts' : scripts}, conf_sec='slurm') config_parser.overwrite_config(config, conf_dict={'precal_scripts' : []}, conf_sec='slurm') config_parser.overwrite_config(config, conf_dict={'postcal_scripts' : []}, conf_sec='slurm') - kwargs = get_config_kwargs(config,'slurm', HPC_DEFAULTS['SLURM_CONFIG_KEYS'.lower()]) + SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()] + HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'.lower()] + kwargs = get_config_kwargs(config,'slurm', SLURM_CONFIG_KEYS) else: scripts = kwargs['scripts'] else: @@ -1455,28 +1464,19 @@ def setup_logger(config,verbose=False): logger.setLevel(loglevel) def main(): - # Define global variables (module level) + # Define global variables global THIS_PROG, SCRIPT_DIR, HPC_DEFAULTS # Define defaults / limits for named HPC facilities THIS_PROG = __file__ SCRIPT_DIR = os.path.dirname(THIS_PROG) DEFAULTS_CONFIG_PATH = "known_hpc.cfg" - HPC_DEFAULTS,_ = config_parser.parse_config( - "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) - ) + HPC_DEFAULTS,_ = config_parser.parse_config("{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH)) #Parse command-line arguments, and setup logger args = parse_args() setup_logger(args.config,args.verbose) - # Select default source - KNOWN_HPC = HPC_DEFAULTS.keys() - if args.cluster in KNOWN_HPC: - HPC_DEFAULTS = HPC_DEFAULTS[args.cluster] - else: - HPC_DEFAULTS = HPC_DEFAULTS['unknown'] - # Mutually exclusive arguments - display version, build config file or run pipeline if args.version: logger.info('This is version {0}'.format(__version__)) From f19dbec3ff7be2d108a436d9d1725930fa4cc8b7 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 13 Oct 2021 12:23:22 +0100 Subject: [PATCH 10/39] Adjusting default parameters --- processMeerKAT/known_hpc.cfg | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 5dcf1ed..2cab872 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -15,6 +15,10 @@ CONFIG = 'default_config.txt' TMP_CONFIG = '.config.tmp' MASTER_SCRIPT = 'submit_pipeline.sh' + PARTITION = 'Main' + MODULES = ['openmpi/2.1.1'] + QOS = 'qos-interactive' + path_binding = '' # Set global values for field, crosscal and SLURM arguments copied to config file, and some of their default values FIELDS_CONFIG_KEYS = [ @@ -22,7 +26,7 @@ 'targetfields','extrafields' ] CROSSCAL_CONFIG_KEYS = [ - 'minbaselines','chanbin','width','timeavg','createmms])', + 'minbaselines','chanbin','width','timeavg','createmms', 'keepmms','spw','nspw','calcrefant','refant','standard', 'badants','badfreqranges' ] @@ -71,7 +75,7 @@ ('quick_tclean.py',True,'') ] # sbatch_file_base must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. - sbatch_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --account={account}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --partition={partition}\n#SBATCH --time={time}" + submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --account={account}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --partition={partition}\n#SBATCH --time={time}" [ilifu] # [DEFAULTS] section defined at the top of this file represents the ilifu configuration. @@ -83,17 +87,21 @@ NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s MEM_PER_NODE_GB_LIMIT = 5000 #237568 MB MEM_PER_NODE_GB_LIMIT_HIGHMEM = 5000 #491520 MB - ACCOUNTS = [] + ACCOUNTS = [''] [galahad] # Specify differences to ilifu - TOTAL_NODES_LIMIT = 1 ### update me! - CPUS_PER_NODE_LIMIT = 16 + TOTAL_NODES_LIMIT = 17 # Total number of hmem nodes + CPUS_PER_NODE_LIMIT = 16 # Number of threads on hmem nodes NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s - MEM_PER_NODE_GB_LIMIT = 1000 #237568 MB ### update me!!! - MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1300 #491520 MB ### update me!!! - ACCOUNTS = ['b03-idia-ag'] ### update me!!! - CONTAINER = '/share/nas/mbowles/mightee/casa-stable.simg' - MPI_WRAPPER = %(CONTAINER)s ### update me!!! # Fairly certain this shouldnt work, but it might. I.e. should probably still reference mpirun somehow. + MEM_PER_NODE_GB_LIMIT = 1500 # GB; 1.5TB available + MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1500 # GB; 1.5 TB availble + ACCOUNTS = [''] # List of allowed accounts; Not currently used: see sbatch_file_base ### update me!!! + CONTAINER = '/share/nas/mbowles/dev/casa-6.simg' # Previously used '/share/nas/mbowles/mightee/casa-stable.simg' + PARTITION = 'CLUSTER' + QOS = 'Normal' + MPI_WRAPPER = 'mpirun' + MODULES = ['openmpi/2.1.1'] + path_binding = '--bind /share:/share ' # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. - sbatch_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}" + submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}" From bd80a5aae4d9b8a6685b33fdf73cc374c76ba9b8 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 13 Oct 2021 12:40:43 +0100 Subject: [PATCH 11/39] Processing known HPC Params --- processMeerKAT/processMeerKAT.py | 61 ++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index b68a68b..abc6a6c 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -130,14 +130,25 @@ def parse_scripts(val): else: return check_path(val) + # Define global variables + global THIS_PROG, SCRIPT_DIR, HPC_DEFAULTS, HPC + THIS_PROG = os.path.realpath(__file__) + SCRIPT_DIR = os.path.dirname(THIS_PROG) + DEFAULTS_CONFIG_PATH = "known_hpc.cfg" + known_hpc_path = "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) + if os.path.isfile(known_hpc_path): + KNOWN_HPCS,_ = config_parser.parse_config(known_hpc_path) + else: + parser.error("Known HPC config file ({0}) not found.".format(known_hpc_path)) + + # Begin parsing parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) parser.add_argument("--hpc",metavar='name', required=False, type=str, default="ilifu", help="Name of hpc facility being used if not known to the config (processMeerKAT/known_hpc.cfg) slurm limits are functionally removed [default: ilifu].") - # Read in default values according to --cluster parameter + # Read in parser default values according to --cluster parameter args, unknown = parser.parse_known_args() - global HPC_DEFAULTS, HPC - HPC = args.hpc if args.hpc in HPC_DEFAULTS.keys() else "unknown" - HPC_DEFAULTS = HPC_DEFAULTS[HPC] + HPC = args.hpc.lower() if args.hpc in KNOWN_HPCS.keys() else "unknown" + HPC_DEFAULTS = KNOWN_HPCS[HPC] parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") parser.add_argument("-C","--config",metavar="path", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") @@ -149,20 +160,20 @@ def parse_scripts(val): help="Distribute tasks of this block size before moving onto next node [default: 1; max: ntasks-per-node].") parser.add_argument("-m","--mem", metavar="num", required=False, type=int, default=HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()], help="Use this many GB of memory (per node) for threadsafe scripts [default: {0}; max: {0}].".format(HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()])) - parser.add_argument("-p","--partition", metavar="name", required=False, type=str, default="Main", help="SLURM partition to use [default: 'Main'].") + parser.add_argument("-p","--partition", metavar="name", required=False, type=str, default=HPC_DEFAULTS['PARTITION'.lower()], help="SLURM partition to use [default: 'Main'].") parser.add_argument("-T","--time", metavar="time", required=False, type=str, default="12:00:00", help="Time limit to use for all jobs, in the form d-hh:mm:ss [default: '12:00:00'].") parser.add_argument("-S","--scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['SCRIPTS'.lower()], help="Run pipeline with these scripts, in this order, using these containers (3rd value - empty string to default to [-c --container]). Is it threadsafe (2nd value)?") parser.add_argument("-b","--precal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()], help="Same as [-S --scripts], but run before calibration.") parser.add_argument("-a","--postcal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['POSTCAL_SCRIPTS'.lower()], help="Same as [-S --scripts], but run after calibration.") - parser.add_argument("--modules", nargs='*', metavar='module', required=False, default=['openmpi/2.1.1'], help="Load these modules within each sbatch script.") + parser.add_argument("--modules", nargs='*', metavar='module', required=False, default=HPC_DEFAULTS['MODULES'.lower()], help="Load these modules within each sbatch script.") parser.add_argument("-w","--mpi_wrapper", metavar="path", required=False, type=str, default=HPC_DEFAULTS['MPI_WRAPPER'.lower()], help="Use this mpi wrapper when calling threadsafe scripts [default: '{0}'].".format(HPC_DEFAULTS['MPI_WRAPPER'.lower()])) parser.add_argument("-c","--container", metavar="path", required=False, type=str, default=HPC_DEFAULTS['CONTAINER'.lower()], help="Use this container when calling scripts [default: '{0}'].".format(HPC_DEFAULTS['CONTAINER'.lower()])) parser.add_argument("-n","--name", metavar="unique", required=False, type=str, default='', help="Unique name to give this pipeline run (e.g. 'run1_'), appended to the start of all job names. [default: ''].") parser.add_argument("-d","--dependencies", metavar="list", required=False, type=str, default='', help="Comma-separated list (without spaces) of SLURM job dependencies (only used when nspw=1). [default: ''].") parser.add_argument("-e","--exclude", metavar="nodes", required=False, type=str, default='', help="SLURM worker nodes to exclude [default: ''].") - parser.add_argument("-A","--account", metavar="group", required=False, type=str, default='b03-idia-ag', help="SLURM accounting group to use (e.g. 'b05-pipelines-ag' - check 'sacctmgr show user $USER cluster=ilifu-slurm20 -s format=account%%30,cluster%%15') [default: 'b03-idia-ag'].") + parser.add_argument("-A","--account", metavar="group", required=False, type=str, default=HPC_DEFAULTS['ACCOUNTS'.lower()][0], help="SLURM accounting group to use (e.g. 'b05-pipelines-ag' - check 'sacctmgr show user $USER cluster=ilifu-slurm20 -s format=account%%30,cluster%%15') [default: 'b03-idia-ag'].") parser.add_argument("-r","--reservation", metavar="name", required=False, type=str, default='', help="SLURM reservation to use. [default: ''].") parser.add_argument("-l","--local", action="store_true", required=False, default=False, help="Build config file locally (i.e. without calling srun) [default: False].") @@ -347,6 +358,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False params['casa_call'] = '' params['casa_log'] = '--nologfile' params['plot_call'] = '' + params['path_binding'] = "{}".format(HPC_DEFAULTS['path_binding']) command = '' params['script'] = check_path(script, update=True) @@ -369,7 +381,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False """ % SPWs.replace(',',' ').replace('0:','') - command += "{mpi_wrapper} singularity exec {container} {plot_call} {casa_call} {script} {args}".format(**params) + command += "{mpi_wrapper} singularity exec {path_binding}{container} {plot_call} {casa_call} {script} {args}".format(**params) if arrayJob: command += '\ncd ..\n' @@ -571,7 +583,7 @@ def write_spw_master(filename,config,SPWs,precal_scripts,postcal_scripts,submit, master.write('\n#Add time as extn to this pipeline run, to give unique filenames') master.write("\nDATE={0}\n".format(timestamp)) master.write('mkdir -p {0}\n'.format(dir)) - master.write('mkdir -p {0}\n\n'.format(LOG_DIR)) + master.write('mkdir -p {0}\n\n'.format(HPC_DEFAULTS['LOG_DIR'.lower()])) extn = '_$DATE.sh' for i,spw in enumerate(SPWs.split(',')): @@ -671,7 +683,7 @@ def write_spw_master(filename,config,SPWs,precal_scripts,postcal_scripts,submit, logger.info('Master script "{0}" written, but will not run.'.format(filename)) -def write_master(filename,config,scripts=[],submit=False,dir='jobScripts',pad_length=5,verbose=False, echo=True, dependencies='',slurm_kwargs={}): +def write_master(filename,config,args,scripts=[],submit=False,dir='jobScripts',pad_length=5,verbose=False, echo=True, dependencies='',slurm_kwargs={}): """Write master pipeline submission script, calling various sbatch files, and writing ancillary job scripts. @@ -871,9 +883,11 @@ def srun(arg_dict,qos=True,time=10,mem=4): call : str srun call with arguments appended.""" - call = 'srun --time={0} --mem={1}GB --partition={2} --account={3}'.format(time,mem,arg_dict['partition'],arg_dict['account']) + call = 'srun --time={0} --mem={1}GB --partition={2}'.format(time,mem,arg_dict['partition']) + if arg_dict['account']!='': + call += ' --account={0}'.format(arg_dict['account']) if qos: - call += ' --qos qos-interactive' + call += ' --qos {0}'.format(HPC_DEFAULTS['qos']) if arg_dict['exclude'] != '': call += ' --exclude={0}'.format(arg_dict['exclude']) if arg_dict['reservation'] != '': @@ -881,7 +895,7 @@ def srun(arg_dict,qos=True,time=10,mem=4): return call -def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], containers=[], num_precal_scripts=0, nodes=8, ntasks_per_node=4, plane=1, partition='Main', +def write_jobs(config, args, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], containers=[], num_precal_scripts=0, nodes=8, ntasks_per_node=4, plane=1, partition='Main', time='12:00:00', submit=False, name='', verbose=False, quiet=False, dependencies='', exclude='', account='b03-idia-ag', reservation='', modules=[], timestamp='', justrun=False): """Write a series of sbatch job files to calibrate a CASA MeasurementSet. @@ -890,6 +904,8 @@ def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], co ---------- config : str Path to config file. + args : obj + Arguments passed to orgiginal command line call of processMeerKAT.py as read by argparse mem : int The memory in GB (per node) to use for this job. mpi_wrapper : str @@ -959,10 +975,10 @@ def write_jobs(config, mpi_wrapper, mem, contents, scripts=[], threadsafe=[], co if crosscal_kwargs['nspw'] > 1: #Build master master script, calling each of the separate SPWs at once, precal scripts before this, and postcal scripts after this - write_spw_master(HPC_DEFAULTS['MASTER_SCRIPT'.lower()],config,SPWs=crosscal_kwargs['spw'],precal_scripts=precal_scripts,postcal_scripts=postcal_scripts,submit=submit,pad_length=pad_length,dependencies=dependencies,timestamp=timestamp,slurm_kwargs=kwargs) + write_spw_master(HPC_DEFAULTS['MASTER_SCRIPT'.lower()],config,args,SPWs=crosscal_kwargs['spw'],precal_scripts=precal_scripts,postcal_scripts=postcal_scripts,submit=submit,pad_length=pad_length,dependencies=dependencies,timestamp=timestamp,slurm_kwargs=kwargs) else: #Build master pipeline submission script - write_master(HPC_DEFAULTS['MASTER_SCRIPT'.lower()],config,scripts=scripts,submit=submit,pad_length=pad_length,verbose=verbose,echo=echo,dependencies=dependencies,slurm_kwargs=kwargs) + write_master(HPC_DEFAULTS['MASTER_SCRIPT'.lower()],config,args,scripts=scripts,submit=submit,pad_length=pad_length,verbose=verbose,echo=echo,dependencies=dependencies,slurm_kwargs=kwargs) def default_config(arg_dict): @@ -1121,7 +1137,7 @@ def format_args(config,submit,quiet,dependencies,justrun): Keyword arguments extracted from [slurm] section of config file, to be passed into write_jobs() function.""" #Ensure all keys exist in these sections - SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE']+HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'] + SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()]+HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'.lower()] kwargs = get_config_kwargs(config,'slurm',SLURM_CONFIG_KEYS) data_kwargs = get_config_kwargs(config,'data',['vis']) get_config_kwargs(config, 'fields', HPC_DEFAULTS['FIELDS_CONFIG_KEYS'.lower()]) @@ -1464,16 +1480,9 @@ def setup_logger(config,verbose=False): logger.setLevel(loglevel) def main(): - # Define global variables - global THIS_PROG, SCRIPT_DIR, HPC_DEFAULTS - - # Define defaults / limits for named HPC facilities - THIS_PROG = __file__ - SCRIPT_DIR = os.path.dirname(THIS_PROG) - DEFAULTS_CONFIG_PATH = "known_hpc.cfg" - HPC_DEFAULTS,_ = config_parser.parse_config("{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH)) - - #Parse command-line arguments, and setup logger + # Parse command-line arguments, and setup logger + # This also creates the global variables: + # THIS_PROG, SCRIPT_DIR, HPC_DEFAULTS, HPC args = parse_args() setup_logger(args.config,args.verbose) From f98309b207b01cfe90a31484917fca12b6be3c46 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 13 Oct 2021 12:52:31 +0100 Subject: [PATCH 12/39] Global params adaptation Bug fixes and --- processMeerKAT/read_ms.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/processMeerKAT/read_ms.py b/processMeerKAT/read_ms.py index 12ed47a..1142c58 100755 --- a/processMeerKAT/read_ms.py +++ b/processMeerKAT/read_ms.py @@ -64,12 +64,16 @@ def get_fields(MS): fieldIDs['targetfields'] = get_field(MS,'TARGET','targetfields',extra_fields,default=default,multiple=True) if 'UNKNOWN' in intents: - try: - polfields = np.array(msmd.namesforfields(msmd.fieldsforintent('UNKNOWN'))) #bogus MeerKAT mislabelling during conversion to MS - for polfield in polfields: - if polfield not in extra_fields: - extra_fields.append(polfield) - except RuntimeError as e: + if len(msmd.fieldsforintent('UNKNOWN')) > 0: + try: + polfields = np.array(msmd.namesforfields(msmd.fieldsforintent('UNKNOWN'))) #bogus MeerKAT mislabelling during conversion to MS + for polfield in polfields: + logger.warning(f"{polfield} not in extra_fields: {polfield not in extra_fields}") + if polfield not in extra_fields: + extra_fields.append(polfield) + except RuntimeError as e: + logger.warning("Intent 'UNKNOWN' present in MS but couldn't find any fields with this intent.") + else: logger.warning("Intent 'UNKNOWN' present in MS but couldn't find any fields with this intent.") #Put any extra fields in extra_fields @@ -210,9 +214,9 @@ def check_scans(MS,nodes,tasks,dopol): tasks = limit while nodes * tasks < limit: - if nodes < processMeerKAT.TOTAL_NODES_LIMIT: + if nodes < HPC_DEFAULTS["TOTAL_NODES_LIMIT".lower()]: nodes += 1 - elif tasks < processMeerKAT.NTASKS_PER_NODE_LIMIT: + elif tasks < HPC_DEFAULTS["NTASKS_PER_NODE_LIMIT".lower()]: tasks += 1 else: break @@ -363,8 +367,20 @@ def get_xy_field(visname, fields): def main(): + # Parse Arguments args = processMeerKAT.parse_args() processMeerKAT.setup_logger(args.config,args.verbose) + + # Read in known_hpc and HPC_DEFAULTS from configuration file. + known_hpc_path = "{0}/{1}".format(os.path.dirname(__file__), "known_hpc.cfg") + if os.path.isfile(known_hpc_path): + KNOWN_HPCS,_ = config_parser.parse_config(known_hpc_path) + else: + parser.error("Known HPC config file ({0}) not found.".format(known_hpc_path)) + global HPC_DEFAULTS + HPC_DEFAULTS = KNOWN_HPCS[args.hpc if args.hpc in KNOWN_HPCS.keys() else "unknown"] + + # Open Measurement Set msmd.open(args.MS) dopol = args.dopol From 7ae1ddb5b9116d2fae3347dfad9a84dc67bb8cf1 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 13 Oct 2021 12:54:56 +0100 Subject: [PATCH 13/39] SPW pipeline run construction Passing command line arguments from initial call to allow for other configurations / calls to be used. --- processMeerKAT/processMeerKAT.py | 56 +++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index abc6a6c..7879bbf 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -516,7 +516,7 @@ def write_sbatch(script,args,mem,mpi_wrapper,contents,nodes=1,tasks=16,name="job logger.debug('Wrote sbatch file "{0}"'.format(sbatch)) -def write_spw_master(filename,config,SPWs,precal_scripts,postcal_scripts,submit,dir='jobScripts',pad_length=5,dependencies='',timestamp='',slurm_kwargs={}): +def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,submit,dir='jobScripts',pad_length=5,dependencies='',timestamp='',slurm_kwargs={}): """Write master master script, which separately calls each of the master scripts in each SPW directory. @@ -524,6 +524,8 @@ def write_spw_master(filename,config,SPWs,precal_scripts,postcal_scripts,submit, Name of master pipeline submission script. config : str Path to config file. + args : obj + Arguments passed to orgiginal command line call of processMeerKAT.py as read by argparse SPWs : str Comma-separated list of spw ranges. precal_scripts : list, optional @@ -667,9 +669,53 @@ def write_spw_master(filename,config,SPWs,precal_scripts,postcal_scripts,submit, os.chmod(filename, 509) #[-R --run] pipeline in each SPW directory to create sbatch files that can be edited - #TODO: fix this hackery! SPW_run_file='out.tmp' - SPW_run_call = """for f in {%s,}; do if [ -d $f ]; then cd $f; %s --config ./%s --run --quiet; cd ..; else echo Directory $f doesn\\'t exist; fi; done""" % (','.join(SPWs.split(',')),os.path.split(THIS_PROG)[1],config) + # Copy argument call parameters made on processMeerKAT.py for this run with minor adjustments + arguments = sys.argv[1:] + for idx, element in enumerate(arguments): + if element in ["-n", "--name"]: + print(element, arguments[idx:]) + if idx+1 < len(arguments): + arguments[idx+1] += "_$f" + elif element in ["-c", "--config"]: + if idx+1 < len(arguments): + arguments[idx+1] = ".config.tmp" + else: + pass + + argument_calls = " ".join(arguments) + if ("-v" or "--verbose") not in argument_calls: + argument_calls += " --quiet" + + # Create script to start processMeerKAT.py for each SPW whilst maintaining args. + SPW_run_file='out.tmp' + SPW_run_call = """ + #!/bin/bash + spws=({spw_array}) + for f in ${{spws[@]}} + do if [ -d $f ] + then + cd $f + source {source} + {program} {argument_calls} + cd {PARENT_DIR} + else + echo Directory $f does not exist + fi + done + """.replace(" ","") + + SPW_run_call = "".join(SPW_run_call).format( + spw_array = " ".join(SPWs.split(',')), + source = os.path.dirname(SCRIPT_DIR)+'/setup.sh', + program = os.path.split(THIS_PROG)[1], + PARENT_DIR = os.path.abspath(os.getcwd()), + argument_calls = argument_calls, + ) + if args.verbose: + logger.info("Explicilty running pipeline on each SPW:{0}".format(SPW_run_call)) + + #processMeerKAT.py run for each SPW with open(SPW_run_file,'w') as out: out.write(SPW_run_call) os.system('bash {0}'.format(SPW_run_file)) @@ -1494,8 +1540,8 @@ def main(): if args.build: default_config(vars(args)) if args.run: - kwargs = format_args(args.config,args.submit,args.quiet,args.dependencies,args.justrun) - write_jobs(args.config, mpi_wrapper=MPI_WRAPPER, contents=HPC_DEFAULTS['sbatch_file_base'], **kwargs) + kwargs = format_args(args.config, args.submit, args.quiet,args. dependencies, args.justrun) + write_jobs(args.config, args, contents=HPC_DEFAULTS['submission_file_base'], **kwargs) if __name__ == "__main__": main() From c193c1185746c99726fc168431b4860e964936ec Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 9 Nov 2021 10:15:42 +0000 Subject: [PATCH 14/39] Minor bug fix --- processMeerKAT/known_hpc.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 2cab872..e9a865c 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -78,7 +78,7 @@ submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --account={account}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --partition={partition}\n#SBATCH --time={time}" [ilifu] - # [DEFAULTS] section defined at the top of this file represents the ilifu configuration. + # [DEFAULT] section defined at the top of this file represents the ilifu configuration. [unknown] # Differences to default: memory / node limits are functionally unlimited. @@ -101,7 +101,7 @@ PARTITION = 'CLUSTER' QOS = 'Normal' MPI_WRAPPER = 'mpirun' - MODULES = ['openmpi/2.1.1'] + MODULES = ['openmpi-2.1.1'] path_binding = '--bind /share:/share ' # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}" From 17e92f2704e35167f3863722b1448b4605fd7578 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 9 Nov 2021 10:16:19 +0000 Subject: [PATCH 15/39] More robust environment adjustment --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index bd056b3..4812ab6 100644 --- a/setup.sh +++ b/setup.sh @@ -1,3 +1,3 @@ -dir=$(dirname $BASH_SOURCE)/processMeerKAT +dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/processMeerKAT" export PATH=$PATH:$dir export PYTHONPATH=$PYTHONPATH:$dir From 0f8e8e011d51bbf0491cd17d401185d70a319b36 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 9 Nov 2021 13:32:38 +0000 Subject: [PATCH 16/39] Maintain cli Maintain cli in recursive processMeerKAT.py calls. --- processMeerKAT/processMeerKAT.py | 42 +++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 0e56591..7d770e7 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -374,6 +374,22 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False params['casa_call'] = 'python' if arrayJob: + # Maintain argument calls for individual SPW runs ### Adapted from `write_spw_master` function. + argv = sys.argv[1:] + for idx, element in enumerate(argv): + # Extend name to include SPW array job is operating on: + if element in ["-n", "--name"]: + if idx+1 < len(argv): + argv[idx+1] += "_${SLURM_ARRAY_JOB_ID}" + # Remove config name. Config is passed into `args` parameter. + elif element in ["-c", "--config"]: + argv[idx] = "" + if idx+1 < len(argv): + argv[idx+1] = "" + else: + pass + params['argument_calls'] = " ".join(argv) + # Iterate over individual SPW folders command += """#Iterate over SPWs in job array, launching one after the other SPWs="%s" arr=($SPWs) @@ -381,7 +397,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False """ % SPWs.replace(',',' ').replace('0:','') - command += "{mpi_wrapper} singularity exec {path_binding}{container} {plot_call} {casa_call} {script} {args}".format(**params) + command += "{mpi_wrapper} singularity exec {path_binding}{container} {plot_call} {casa_call} {script} {args}{argument_calls}".format(**params) if arrayJob: command += '\ncd ..\n' @@ -588,10 +604,29 @@ def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,su master.write('mkdir -p {0}\n\n'.format(HPC_DEFAULTS['LOG_DIR'.lower()])) extn = '_$DATE.sh' + # Maintain argument calls for individual SPW runs ### Adapted from `write_spw_master` function. + argv = sys.argv[1:] + ignore = ["--run", "-R", ] + for idx, element in enumerate(argv): + # Extend name to include SPW array job is operating on: + if element in ["-n", "--name"]: + if idx+1 < len(argv): + argv[idx+1] += "_${SLURM_ARRAY_JOB_ID}" + # Remove config name. Config is passed into `args` parameter. + elif element in ["-c", "--config"]: + argv[idx] = "" + if idx+1 < len(argv): + argv[idx+1] = "" + else: + pass + argument_calls = " ".join(argv) + if ("-v" or "--verbose") not in argument_calls: + argument_calls += " --quiet" + for i,spw in enumerate(SPWs.split(',')): master.write('echo Running pipeline in directory "{0}" for spectral window 0:{0}\n'.format(spw)) master.write('cd {0}\n'.format(spw)) - master.write('output=$({0} --config ./{1} --run --submit --quiet --justrun'.format(os.path.split(THIS_PROG)[1],config)) + master.write('output=$({0} --config ./{config} --run --submit --justrun {argument_calls}'.format(os.path.split(THIS_PROG)[1], config=config, argument_calls=argument_calls)) if partition: master.write(' --dependencies=$partitionID\_{0}'.format(i)) elif len(precal_scripts) > 0: @@ -1548,9 +1583,10 @@ def main(): if args.license: logger.info(license) if args.build: + # Write default config according to building parameters. default_config(vars(args)) if args.run: - kwargs = format_args(args.config, args.submit, args.quiet,args. dependencies, args.justrun) + kwargs = format_args(args.config, args.submit, args.quiet, args.dependencies, args.justrun) write_jobs(args.config, args, contents=HPC_DEFAULTS['submission_file_base'], **kwargs) if __name__ == "__main__": From 29fd36ecbe1b77f1e4ca2ff0bfc1a9d5c85ebfd2 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:03:10 +0000 Subject: [PATCH 17/39] Galahad specific mpi call change Added call parameters to avoid que pair (QP) errors. --- processMeerKAT/known_hpc.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index e9a865c..06c3b53 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -100,7 +100,7 @@ CONTAINER = '/share/nas/mbowles/dev/casa-6.simg' # Previously used '/share/nas/mbowles/mightee/casa-stable.simg' PARTITION = 'CLUSTER' QOS = 'Normal' - MPI_WRAPPER = 'mpirun' + MPI_WRAPPER = 'mpirun --mca oob tcp' MODULES = ['openmpi-2.1.1'] path_binding = '--bind /share:/share ' # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. From 48c45c0c8cda88d614f60050da45f1a3f42758a2 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:05:09 +0000 Subject: [PATCH 18/39] Bug fix: recursive CLI calls --- processMeerKAT/processMeerKAT.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 7d770e7..60a44c0 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -359,6 +359,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False params['casa_log'] = '--nologfile' params['plot_call'] = '' params['path_binding'] = "{}".format(HPC_DEFAULTS['path_binding']) + params['argument_calls'] = '' command = '' params['script'] = check_path(script, update=True) @@ -388,7 +389,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False argv[idx+1] = "" else: pass - params['argument_calls'] = " ".join(argv) + params['argument_calls'] = " "+" ".join(argv) # Iterate over individual SPW folders command += """#Iterate over SPWs in job array, launching one after the other SPWs="%s" From a4ac6ddf34cf1ec293636caf21636f67304f9595 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:07:46 +0000 Subject: [PATCH 19/39] Reintroduce global script paths --- processMeerKAT/processMeerKAT.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 60a44c0..51a853d 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -32,10 +32,13 @@ import logging from time import gmtime from datetime import datetime + logging.Formatter.converter = gmtime logger = logging.getLogger(__name__) logging.basicConfig(format="%(asctime)-15s %(levelname)s: %(message)s") +THIS_PROG = os.path.realpath(__file__) +SCRIPT_DIR = os.path.dirname(THIS_PROG) def check_path(path,update=False): @@ -131,24 +134,24 @@ def parse_scripts(val): return check_path(val) # Define global variables - global THIS_PROG, SCRIPT_DIR, HPC_DEFAULTS, HPC - THIS_PROG = os.path.realpath(__file__) - SCRIPT_DIR = os.path.dirname(THIS_PROG) + global HPC_DEFAULTS, HPC_NAME, HPC_CONFIG + DEFAULTS_CONFIG_PATH = "known_hpc.cfg" known_hpc_path = "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) + if os.path.isfile(known_hpc_path): - KNOWN_HPCS,_ = config_parser.parse_config(known_hpc_path) + KNOWN_HPCS, HPC_CONFIG = config_parser.parse_config(known_hpc_path) else: parser.error("Known HPC config file ({0}) not found.".format(known_hpc_path)) # Begin parsing parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) - parser.add_argument("--hpc",metavar='name', required=False, type=str, default="ilifu", help="Name of hpc facility being used if not known to the config (processMeerKAT/known_hpc.cfg) slurm limits are functionally removed [default: ilifu].") + parser.add_argument("--hpc",metavar='name', required=False, type=str, default="ilifu", help="Name of hpc facility being used. If not known to processMeerKAT/known_hpc.cfg slurm limits are functionally removed [default: ilifu].") # Read in parser default values according to --cluster parameter args, unknown = parser.parse_known_args() - HPC = args.hpc.lower() if args.hpc in KNOWN_HPCS.keys() else "unknown" - HPC_DEFAULTS = KNOWN_HPCS[HPC] + HPC_NAME = args.hpc.lower() if args.hpc in KNOWN_HPCS.keys() else "unknown" + HPC_DEFAULTS = KNOWN_HPCS[HPC_NAME] parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") parser.add_argument("-C","--config",metavar="path", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") @@ -261,7 +264,7 @@ def validate_args(args,config,parser=None): msg = "Only input an MS [-M --MS] during [-B --build] step. Otherwise input is ignored." raise_error(config, msg, parser) - if HPC=="unknown": + if HPC_NAME=="unknown": msg = "HPC facility [--hpc] is not in 'known_hpc.cfg', reverting to 'unknown' HPC. You input {0}. Pipeline will rely entirely on the specified arguemnts. No upper limits will be set. HPC specific selections within your config may cause pipeline runs to fail!" logger.warning(msg.format(args['hpc'])) @@ -1574,7 +1577,7 @@ def setup_logger(config,verbose=False): def main(): # Parse command-line arguments, and setup logger # This also creates the global variables: - # THIS_PROG, SCRIPT_DIR, HPC_DEFAULTS, HPC + # HPC_DEFAULTS, HPC_NAME, HPC_CONFIG args = parse_args() setup_logger(args.config,args.verbose) From 749e5bea876b0eb69556d65a5322fbc5fa8e560c Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:27:56 +0000 Subject: [PATCH 20/39] Saving HPC parameter Saving selected HPC name in config during build to allow for future use without setting hpc explicitly in CLI. i.e. for running. --- processMeerKAT/processMeerKAT.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 51a853d..793f037 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -155,6 +155,18 @@ def parse_scripts(val): parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") parser.add_argument("-C","--config",metavar="path", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") + + # Extract hpc name used during build and warn if not the same as CLI hpc + config_dict, config = config_parser.parse_config(args.config) + if config.has_option('run', 'hpc'): + config_hpc_name = config['run']['hpc'] + else: + config_hpc_name = HPC_NAME + if HPC_NAME != config_hpc_name: + msg = "Entered hpc ({HPC_NAME}) is not the same as was used to build this pipeline instance! '{config_hpc_name}' was used. Pipeline may not function as expected. Please consider rebuilding pipeline with correct hpc selection." + logger.warning(msg.format(HPC_NAME=HPC_NAME, config_hpc_name=config_hpc_name)) + + # Parse in remaining arguments parser.add_argument("-N","--nodes",metavar="num", required=False, type=int, default=1, help="Use this number of nodes [default: 1; max: {0}].".format(HPC_DEFAULTS['TOTAL_NODES_LIMIT'.lower()])) parser.add_argument("-t","--ntasks-per-node", metavar="num", required=False, type=int, default=8, @@ -1090,9 +1102,11 @@ def default_config(arg_dict): #Overwrite CL parameters in config under section [slurm] config_parser.overwrite_config(filename, conf_dict=slurm_dict, conf_sec='slurm') - #Add MS to config file under section [data] and dopol under section [run] + #Add MS to config file under section [data] config_parser.overwrite_config(filename, conf_dict={'vis' : "'{0}'".format(MS)}, conf_sec='data') - config_parser.overwrite_config(filename, conf_dict={'dopol' : arg_dict['dopol']}, conf_sec='run', sec_comment='# Internal variables for pipeline execution') + # Add dopol and hpc under section [run] + run_dict = {'dopol' : arg_dict['dopol'], 'hpc' : "'{}'".format(arg_dict['hpc'])} + config_parser.overwrite_config(filename, conf_dict=run_dict, conf_sec='run', sec_comment='# Internal variables for pipeline execution') if not arg_dict['do2GC'] or not arg_dict['science_image']: remove_scripts = [] From 6c91c764894e4e94582f306c8ff0813cf323f9c3 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:40:00 +0000 Subject: [PATCH 21/39] Path correction to allow relative imports Importing `processMeerKAT.py` and `config_parser.py` from the parent directory has caused many issues. Using this fix is not ideal, but it should be a consistent fix across systems. --- processMeerKAT/aux_scripts/concat.py | 4 ++++ processMeerKAT/crosscal_scripts/calc_refant.py | 10 +++++++--- processMeerKAT/crosscal_scripts/flag_round_1.py | 4 ++++ processMeerKAT/crosscal_scripts/flag_round_2.py | 5 +++++ processMeerKAT/crosscal_scripts/partition.py | 5 +++++ processMeerKAT/crosscal_scripts/plot_solutions.py | 6 ++++++ processMeerKAT/crosscal_scripts/plotcal_spw.py | 6 ++++++ processMeerKAT/crosscal_scripts/quick_tclean.py | 4 ++++ processMeerKAT/crosscal_scripts/setjy.py | 4 ++++ processMeerKAT/crosscal_scripts/split.py | 4 ++++ processMeerKAT/crosscal_scripts/xx_yy_apply.py | 4 ++++ processMeerKAT/crosscal_scripts/xx_yy_solve.py | 4 ++++ processMeerKAT/crosscal_scripts/xy_yx_apply.py | 4 ++++ processMeerKAT/crosscal_scripts/xy_yx_solve.py | 4 ++++ processMeerKAT/selfcal_scripts/selfcal_part1.py | 4 ++++ processMeerKAT/selfcal_scripts/selfcal_part2.py | 4 ++++ processMeerKAT/selfcal_scripts/set_sky_model.py | 6 ++++++ 17 files changed, 79 insertions(+), 3 deletions(-) diff --git a/processMeerKAT/aux_scripts/concat.py b/processMeerKAT/aux_scripts/concat.py index 231c3ed..fe119f8 100644 --- a/processMeerKAT/aux_scripts/concat.py +++ b/processMeerKAT/aux_scripts/concat.py @@ -6,6 +6,10 @@ import glob from shutil import copytree +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser from config_parser import validate_args as va import bookkeeping diff --git a/processMeerKAT/crosscal_scripts/calc_refant.py b/processMeerKAT/crosscal_scripts/calc_refant.py index 3957142..949bb88 100644 --- a/processMeerKAT/crosscal_scripts/calc_refant.py +++ b/processMeerKAT/crosscal_scripts/calc_refant.py @@ -4,13 +4,17 @@ """ Calculates the reference antenna """ +import os, sys +import numpy as np + +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser from config_parser import validate_args as va import bookkeeping -import os -import numpy as np - from casatasks import * logfile=casalog.logfile() casalog.setlogfile('logs/{SLURM_JOB_NAME}-{SLURM_JOB_ID}.casa'.format(**os.environ)) diff --git a/processMeerKAT/crosscal_scripts/flag_round_1.py b/processMeerKAT/crosscal_scripts/flag_round_1.py index 5b77416..8671836 100644 --- a/processMeerKAT/crosscal_scripts/flag_round_1.py +++ b/processMeerKAT/crosscal_scripts/flag_round_1.py @@ -4,6 +4,10 @@ import sys import os +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser from config_parser import validate_args as va import bookkeeping diff --git a/processMeerKAT/crosscal_scripts/flag_round_2.py b/processMeerKAT/crosscal_scripts/flag_round_2.py index 9708497..df8e5e0 100644 --- a/processMeerKAT/crosscal_scripts/flag_round_2.py +++ b/processMeerKAT/crosscal_scripts/flag_round_2.py @@ -4,6 +4,11 @@ import sys import os +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + + import config_parser from config_parser import validate_args as va import bookkeeping diff --git a/processMeerKAT/crosscal_scripts/partition.py b/processMeerKAT/crosscal_scripts/partition.py index df1d2c3..862adc3 100644 --- a/processMeerKAT/crosscal_scripts/partition.py +++ b/processMeerKAT/crosscal_scripts/partition.py @@ -7,6 +7,11 @@ import sys import os +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + + import config_parser from config_parser import validate_args as va import read_ms diff --git a/processMeerKAT/crosscal_scripts/plot_solutions.py b/processMeerKAT/crosscal_scripts/plot_solutions.py index 2481cab..05a730f 100644 --- a/processMeerKAT/crosscal_scripts/plot_solutions.py +++ b/processMeerKAT/crosscal_scripts/plot_solutions.py @@ -1,7 +1,13 @@ #Copyright (C) 2020 Inter-University Institute for Data Intensive Astronomy #See processMeerKAT.py for license details. +import sys import os + +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser from config_parser import validate_args as va import bookkeeping diff --git a/processMeerKAT/crosscal_scripts/plotcal_spw.py b/processMeerKAT/crosscal_scripts/plotcal_spw.py index 7541e84..64f29dd 100644 --- a/processMeerKAT/crosscal_scripts/plotcal_spw.py +++ b/processMeerKAT/crosscal_scripts/plotcal_spw.py @@ -1,6 +1,12 @@ #!/usr/bin/env python3 +import sys import os + +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import glob import config_parser import traceback diff --git a/processMeerKAT/crosscal_scripts/quick_tclean.py b/processMeerKAT/crosscal_scripts/quick_tclean.py index 3443902..13b0904 100644 --- a/processMeerKAT/crosscal_scripts/quick_tclean.py +++ b/processMeerKAT/crosscal_scripts/quick_tclean.py @@ -4,6 +4,10 @@ import sys import os +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser from config_parser import validate_args as va import bookkeeping diff --git a/processMeerKAT/crosscal_scripts/setjy.py b/processMeerKAT/crosscal_scripts/setjy.py index 5fd8e62..23de5e4 100644 --- a/processMeerKAT/crosscal_scripts/setjy.py +++ b/processMeerKAT/crosscal_scripts/setjy.py @@ -3,6 +3,10 @@ import os, sys, shutil +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import bookkeeping from config_parser import validate_args as va import numpy as np diff --git a/processMeerKAT/crosscal_scripts/split.py b/processMeerKAT/crosscal_scripts/split.py index ba0c706..f021691 100644 --- a/processMeerKAT/crosscal_scripts/split.py +++ b/processMeerKAT/crosscal_scripts/split.py @@ -4,6 +4,10 @@ import sys import os +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser import bookkeeping from config_parser import validate_args as va diff --git a/processMeerKAT/crosscal_scripts/xx_yy_apply.py b/processMeerKAT/crosscal_scripts/xx_yy_apply.py index 9d6a4aa..7275d26 100644 --- a/processMeerKAT/crosscal_scripts/xx_yy_apply.py +++ b/processMeerKAT/crosscal_scripts/xx_yy_apply.py @@ -4,6 +4,10 @@ import sys import os +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser import bookkeeping from config_parser import validate_args as va diff --git a/processMeerKAT/crosscal_scripts/xx_yy_solve.py b/processMeerKAT/crosscal_scripts/xx_yy_solve.py index b79d5c8..7499d16 100644 --- a/processMeerKAT/crosscal_scripts/xx_yy_solve.py +++ b/processMeerKAT/crosscal_scripts/xx_yy_solve.py @@ -5,6 +5,10 @@ import os import shutil +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser import bookkeeping from config_parser import validate_args as va diff --git a/processMeerKAT/crosscal_scripts/xy_yx_apply.py b/processMeerKAT/crosscal_scripts/xy_yx_apply.py index 132525a..0788efb 100644 --- a/processMeerKAT/crosscal_scripts/xy_yx_apply.py +++ b/processMeerKAT/crosscal_scripts/xy_yx_apply.py @@ -5,6 +5,10 @@ import os import shutil +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser import bookkeeping, read_ms from config_parser import validate_args as va diff --git a/processMeerKAT/crosscal_scripts/xy_yx_solve.py b/processMeerKAT/crosscal_scripts/xy_yx_solve.py index 780aea1..5040702 100644 --- a/processMeerKAT/crosscal_scripts/xy_yx_solve.py +++ b/processMeerKAT/crosscal_scripts/xy_yx_solve.py @@ -5,6 +5,10 @@ import os import shutil +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser import bookkeeping from config_parser import validate_args as va diff --git a/processMeerKAT/selfcal_scripts/selfcal_part1.py b/processMeerKAT/selfcal_scripts/selfcal_part1.py index 5360c59..bbfea95 100644 --- a/processMeerKAT/selfcal_scripts/selfcal_part1.py +++ b/processMeerKAT/selfcal_scripts/selfcal_part1.py @@ -6,6 +6,10 @@ import os import re +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser from config_parser import validate_args as va import bookkeeping diff --git a/processMeerKAT/selfcal_scripts/selfcal_part2.py b/processMeerKAT/selfcal_scripts/selfcal_part2.py index dcbb476..b1f7cbd 100644 --- a/processMeerKAT/selfcal_scripts/selfcal_part2.py +++ b/processMeerKAT/selfcal_scripts/selfcal_part2.py @@ -7,6 +7,10 @@ import os import re +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import config_parser from config_parser import validate_args as va import bookkeeping diff --git a/processMeerKAT/selfcal_scripts/set_sky_model.py b/processMeerKAT/selfcal_scripts/set_sky_model.py index 3c0bc6c..ddba93e 100644 --- a/processMeerKAT/selfcal_scripts/set_sky_model.py +++ b/processMeerKAT/selfcal_scripts/set_sky_model.py @@ -1,7 +1,13 @@ #Copyright (C) 2020 Inter-University Institute for Data Intensive Astronomy #See processMeerKAT.py for license details. +import sys import os + +# Adapt PYTHONPATH to include processMeerKAT +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.dirname(SCRIPT_DIR)) + import bookkeeping from selfcal_scripts.selfcal_part2 import find_outliers from casatasks import casalog From ef501503548de2a6ef99aa815fc59c9da89aa0a5 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:40:23 +0000 Subject: [PATCH 22/39] Minor change to remove additional dependancy. --- processMeerKAT/config_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processMeerKAT/config_parser.py b/processMeerKAT/config_parser.py index ba61291..d50038d 100755 --- a/processMeerKAT/config_parser.py +++ b/processMeerKAT/config_parser.py @@ -13,7 +13,7 @@ def parse_args(): Parse the command line arguments """ parser = argparse.ArgumentParser() - parser.add_argument('-C','--config', default=processMeerKAT.CONFIG, required=False, help='Name of the input config file') + parser.add_argument('-C','--config', default="myconfig.txt", required=False, help='Name of the input config file') args, __ = parser.parse_known_args() From 30b31c3e6e4c0a342df31b02cce3251d1cd7c360 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:45:44 +0000 Subject: [PATCH 23/39] Removing global dependency Using hpc option saved in built config to remove global dependency by reading in known hpc config. --- processMeerKAT/crosscal_scripts/partition.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/processMeerKAT/crosscal_scripts/partition.py b/processMeerKAT/crosscal_scripts/partition.py index 862adc3..c54f73d 100644 --- a/processMeerKAT/crosscal_scripts/partition.py +++ b/processMeerKAT/crosscal_scripts/partition.py @@ -52,6 +52,13 @@ def main(args,taskvals): include_crosshand = va(taskvals, 'run', 'dopol', bool, default=False) createmms = va(taskvals, 'crosscal', 'createmms', bool, default=True) + # HPC Specific Configuration + known_hpc_path = os.path.dirname(SCRIPT_DIR)+"/known_hpc.cfg" + KNOWN_HPCS, HPC_CONFIG = config_parser.parse_config(known_hpc_path) + HPC_NAME = taskvals["run"]["hpc"] + HPC_NAME = HPC_NAME if HPC_NAME in KNOWN_HPCS.keys() else "unknown" + CPUS_PER_NODE_LIMIT = va(KNOWN_HPCS, HPC_NAME, "CPUS_PER_NODE_LIMIT".lower(), dtype=int) + if nspw > 1: casalog.setlogfile('logs/{SLURM_JOB_NAME}-{SLURM_ARRAY_JOB_ID}_{SLURM_ARRAY_TASK_ID}.casa'.format(**os.environ)) else: @@ -69,7 +76,7 @@ def main(args,taskvals): if not include_crosshand and npol == 4: npol = 2 - CPUs = npol if tasks*npol <= processMeerKAT.CPUS_PER_NODE_LIMIT else 1 #hard-code for number of polarisations + CPUs = npol if tasks*npol <= CPUS_PER_NODE_LIMIT else 1 #hard-code for number of polarisations mvis = do_partition(visname, spw, preavg, CPUs, include_crosshand, createmms, spwname) mvis = "'{0}'".format(mvis) From 85a5cb45e035f63c62867922d14efe5f88347363 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Nov 2021 15:46:08 +0000 Subject: [PATCH 24/39] typo --- processMeerKAT/processMeerKAT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 793f037..da43c89 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -467,7 +467,7 @@ def write_sbatch(script,args,mem,mpi_wrapper,contents,nodes=1,tasks=16,name="job SLURM reservation to use. modules : list, optional Modules to load upon execution of sbatch script. - justrun : bool, optionall + justrun : bool, optional Just run the pipeline without rebuilding each job script (if it exists).""" if not os.path.exists(HPC_DEFAULTS['LOG_DIR'.lower()]): From fdf07e826fee29b65feed3da7820ca15b474eea7 Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Mon, 31 Jan 2022 19:31:52 +1100 Subject: [PATCH 25/39] Add petrichor --- processMeerKAT/known_hpc.cfg | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 06c3b53..114a2a6 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -105,3 +105,20 @@ path_binding = '--bind /share:/share ' # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}" + +[petrichor] + # Specify differences to ilifu + TOTAL_NODES_LIMIT = 110 + CPUS_PER_NODE_LIMIT = 64 + NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s + MEM_PER_NODE_GB_LIMIT = 512 # GB + MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1000 # + ACCOUNTS = [''] # List of allowed accounts; Not currently used: see sbatch_file_base ### update me!!! + CONTAINER = '/scratch1/tho822/containers/casa-pipeline/casa-6.1.2.7-modular.simg' # + PARTITION = 'defq' + QOS = 'express' + MPI_WRAPPER = 'mpirun' + MODULES = ['singularity','openmpi'] + path_binding = '--bind /scratch1:/scratch1,/scratch2:/scratch2 ' + # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. + submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}" From ae4fc5f84802429475732e301f5ea8d9daf7fd18 Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Mon, 31 Jan 2022 19:32:05 +1100 Subject: [PATCH 26/39] Fix parsing --- processMeerKAT/processMeerKAT.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index da43c89..5d55b49 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -139,13 +139,15 @@ def parse_scripts(val): DEFAULTS_CONFIG_PATH = "known_hpc.cfg" known_hpc_path = "{0}/{1}".format(SCRIPT_DIR, DEFAULTS_CONFIG_PATH) + # Begin parsing + parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) + if os.path.isfile(known_hpc_path): KNOWN_HPCS, HPC_CONFIG = config_parser.parse_config(known_hpc_path) else: parser.error("Known HPC config file ({0}) not found.".format(known_hpc_path)) - # Begin parsing - parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) + parser.add_argument("--hpc",metavar='name', required=False, type=str, default="ilifu", help="Name of hpc facility being used. If not known to processMeerKAT/known_hpc.cfg slurm limits are functionally removed [default: ilifu].") # Read in parser default values according to --cluster parameter @@ -154,12 +156,13 @@ def parse_scripts(val): HPC_DEFAULTS = KNOWN_HPCS[HPC_NAME] parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") - parser.add_argument("-C","--config",metavar="path", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") + parser.add_argument("-C","--config",metavar="config", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") + args, unknown = parser.parse_known_args() # Extract hpc name used during build and warn if not the same as CLI hpc config_dict, config = config_parser.parse_config(args.config) if config.has_option('run', 'hpc'): - config_hpc_name = config['run']['hpc'] + config_hpc_name = config['run']['hpc'].strip("'") else: config_hpc_name = HPC_NAME if HPC_NAME != config_hpc_name: @@ -1135,7 +1138,7 @@ def default_config(arg_dict): mpi_wrapper = srun(arg_dict) #Write and submit srun command to extract fields, and insert them into config file under section [fields] - params = '-B -M {MS} -C {config} -N {nodes} -t {ntasks_per_node}'.format(**arg_dict) + params = '-B -M {MS} -C {config} -N {nodes} -t {ntasks_per_node} --hpc {hpc}'.format(**arg_dict) if arg_dict['dopol']: params += ' -P' if arg_dict['verbose']: From 0c2e99840722ae3a1b6bf4d545933cd3f7907c03 Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Mon, 31 Jan 2022 20:27:42 +1100 Subject: [PATCH 27/39] Fix missing kwargs --- processMeerKAT/processMeerKAT.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 5d55b49..def7e96 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -1265,7 +1265,7 @@ def format_args(config,submit,quiet,dependencies,justrun): #Check selfcal params if config_parser.has_section(config,'selfcal'): - selfcal_kwargs = get_config_kwargs(config, 'selfcal', SELFCAL_CONFIG_KEYS) + selfcal_kwargs = get_config_kwargs(config, 'selfcal', HPC_DEFAULTS['SELFCAL_CONFIG_KEYS'.lower()]) params = bookkeeping.get_selfcal_params() if selfcal_kwargs['loop'] > 0: logger.warning("Starting with loop={0}, which is only valid if previous loops were successfully run in this directory.".format(selfcal_kwargs['loop'])) @@ -1281,7 +1281,7 @@ def format_args(config,submit,quiet,dependencies,justrun): os.system(command) if config_parser.has_section(config,'image'): - imaging_kwargs = get_config_kwargs(config, 'image', IMAGING_CONFIG_KEYS) + imaging_kwargs = get_config_kwargs(config, 'image', HPC_DEFAULTS['IMAGING_CONFIG_KEYS'.lower()]) #If nspw = 1 and precal or postcal scripts present, overwrite config and reload if nspw == 1: From a27429443022d249ba9b638471ae536931425a01 Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Mon, 31 Jan 2022 20:29:48 +1100 Subject: [PATCH 28/39] Ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b1d66dc..16180c9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ build/ dist/ venv/ +.vscode/settings.json +processMeerKAT/workspace.code-workspace From d6cdf886463f325ed06c79333ad2ca17ebf0f86a Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Fri, 4 Feb 2022 15:12:00 +1100 Subject: [PATCH 29/39] Capital C bug --- processMeerKAT/processMeerKAT.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index def7e96..3dcbac3 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -401,7 +401,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False if idx+1 < len(argv): argv[idx+1] += "_${SLURM_ARRAY_JOB_ID}" # Remove config name. Config is passed into `args` parameter. - elif element in ["-c", "--config"]: + elif element in ["-C", "--config"]: argv[idx] = "" if idx+1 < len(argv): argv[idx+1] = "" @@ -632,7 +632,7 @@ def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,su if idx+1 < len(argv): argv[idx+1] += "_${SLURM_ARRAY_JOB_ID}" # Remove config name. Config is passed into `args` parameter. - elif element in ["-c", "--config"]: + elif element in ["-C", "--config"]: argv[idx] = "" if idx+1 < len(argv): argv[idx+1] = "" @@ -731,7 +731,7 @@ def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,su print(element, arguments[idx:]) if idx+1 < len(arguments): arguments[idx+1] += "_$f" - elif element in ["-c", "--config"]: + elif element in ["-C", "--config"]: if idx+1 < len(arguments): arguments[idx+1] = ".config.tmp" else: @@ -1375,8 +1375,11 @@ def format_args(config,submit,quiet,dependencies,justrun): #sys.exit(1) #If everything up until here has passed, we can copy config file to TMP_CONFIG (in case user runs sbatch manually) and inform user - logger.debug("Copying '{0}' to '{1}', and using this to run pipeline.".format(config, HPC_DEFAULTS['TMP_CONFIG'.lower()])) - copyfile(config, HPC_DEFAULTS['TMP_CONFIG'.lower()]) + # Skip if config is temporary + if config == HPC_DEFAULTS['TMP_CONFIG'.lower()]: + logger.debug("Not copying '{0}' to '{1}'. They're the same file.".format(config, HPC_DEFAULTS['TMP_CONFIG'.lower()])) + else: + logger.debug("Copying '{0}' to '{1}', and using this to run pipeline.".format(config, HPC_DEFAULTS['TMP_CONFIG'.lower()])) if not quiet: logger.warning("Changing [slurm] section in your config will have no effect unless you [-R --run] again.") From 8b3afbea4de7f61652164e9cbbd8954bb3543883 Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Fri, 4 Feb 2022 16:19:11 +1100 Subject: [PATCH 30/39] Fix argument logic --- processMeerKAT/processMeerKAT.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 3dcbac3..fc090f2 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -639,7 +639,7 @@ def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,su else: pass argument_calls = " ".join(argv) - if ("-v" or "--verbose") not in argument_calls: + if not any((arg in argument_calls for arg in ("-v", "--verbose"))): argument_calls += " --quiet" for i,spw in enumerate(SPWs.split(',')): @@ -738,7 +738,7 @@ def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,su pass argument_calls = " ".join(arguments) - if ("-v" or "--verbose") not in argument_calls: + if not any((arg in argument_calls for arg in ("-v", "--verbose"))): argument_calls += " --quiet" # Create script to start processMeerKAT.py for each SPW whilst maintaining args. From 23cbf8b2fb704060a4601be2a27958fe99dfb1a2 Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Sat, 2 Jul 2022 16:19:19 +1000 Subject: [PATCH 31/39] Add spw prefix --- processMeerKAT/known_hpc.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 114a2a6..574a856 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -15,6 +15,7 @@ CONFIG = 'default_config.txt' TMP_CONFIG = '.config.tmp' MASTER_SCRIPT = 'submit_pipeline.sh' + SPW_PREFIX = '*:' PARTITION = 'Main' MODULES = ['openmpi/2.1.1'] QOS = 'qos-interactive' From 767e6a76bbef44e88f5483fac87a43a1c88041ed Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Sat, 2 Jul 2022 16:26:22 +1000 Subject: [PATCH 32/39] Fix overdetermined args --- processMeerKAT/processMeerKAT.py | 103 ++++++++++++++++++------------- 1 file changed, 60 insertions(+), 43 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index 37d3910..cbc73f9 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -149,49 +149,35 @@ def parse_scripts(val): - parser.add_argument("--hpc",metavar='name', required=False, type=str, default="ilifu", help="Name of hpc facility being used. If not known to processMeerKAT/known_hpc.cfg slurm limits are functionally removed [default: ilifu].") - # Read in parser default values according to --cluster parameter - args, unknown = parser.parse_known_args() - HPC_NAME = args.hpc.lower() if args.hpc in KNOWN_HPCS.keys() else "unknown" - HPC_DEFAULTS = KNOWN_HPCS[HPC_NAME] + parser.add_argument("--hpc",metavar='name', required=False, type=str, default="unkown", help="Name of hpc facility being used. If not known to processMeerKAT/known_hpc.cfg slurm limits are functionally removed [default: ilifu].") - parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") - parser.add_argument("-C","--config",metavar="config", default=HPC_DEFAULTS['CONFIG'.lower()], required=False, type=str, help="Relative (not absolute) path to config file.") - args, unknown = parser.parse_known_args() - # Extract hpc name used during build and warn if not the same as CLI hpc - config_dict, config = config_parser.parse_config(args.config) - if config.has_option('run', 'hpc'): - config_hpc_name = config['run']['hpc'].strip("'") - else: - config_hpc_name = HPC_NAME - if HPC_NAME != config_hpc_name: - msg = "Entered hpc ({HPC_NAME}) is not the same as was used to build this pipeline instance! '{config_hpc_name}' was used. Pipeline may not function as expected. Please consider rebuilding pipeline with correct hpc selection." - logger.warning(msg.format(HPC_NAME=HPC_NAME, config_hpc_name=config_hpc_name)) + parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") + parser.add_argument("-C","--config",metavar="config", default=None, required=False, type=str, help="Relative (not absolute) path to config file.") # Parse in remaining arguments - parser.add_argument("-N","--nodes",metavar="num", required=False, type=int, default=1, - help="Use this number of nodes [default: 1; max: {0}].".format(HPC_DEFAULTS['TOTAL_NODES_LIMIT'.lower()])) - parser.add_argument("-t","--ntasks-per-node", metavar="num", required=False, type=int, default=8, - help="Use this number of tasks (per node) [default: 16; max: {0}].".format(HPC_DEFAULTS['NTASKS_PER_NODE_LIMIT'.lower()])) + parser.add_argument("-N","--nodes",metavar="num", required=False, type=int, default=None, + help="Use this number of nodes [default: 1].") + parser.add_argument("-t","--ntasks-per-node", metavar="num", required=False, type=int, default=None, + help="Use this number of tasks (per node) [default: 8]") parser.add_argument("-D","--plane", metavar="num", required=False, type=int, default=1, help="Distribute tasks of this block size before moving onto next node [default: 1; max: ntasks-per-node].") - parser.add_argument("-m","--mem", metavar="num", required=False, type=int, default=HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()], - help="Use this many GB of memory (per node) for threadsafe scripts [default: {0}; max: {0}].".format(HPC_DEFAULTS['MEM_PER_NODE_GB_LIMIT'.lower()])) - parser.add_argument("-p","--partition", metavar="name", required=False, type=str, default=HPC_DEFAULTS['PARTITION'.lower()], help="SLURM partition to use [default: 'Main'].") + parser.add_argument("-m","--mem", metavar="num", required=False, type=int, default=None, + help="Use this many GB of memory (per node) for threadsafe scripts [default: {0}; max: {0}].") + parser.add_argument("-p","--partition", metavar="name", required=False, type=str, default=None, help="SLURM partition to use [default: 'Main'].") parser.add_argument("-T","--time", metavar="time", required=False, type=str, default="12:00:00", help="Time limit to use for all jobs, in the form d-hh:mm:ss [default: '12:00:00'].") - parser.add_argument("-S","--scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['SCRIPTS'.lower()], + parser.add_argument("-S","--scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=None, help="Run pipeline with these scripts, in this order, using these containers (3rd value - empty string to default to [-c --container]). Is it threadsafe (2nd value)?") - parser.add_argument("-b","--precal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['PRECAL_SCRIPTS'.lower()], help="Same as [-S --scripts], but run before calibration.") - parser.add_argument("-a","--postcal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=HPC_DEFAULTS['POSTCAL_SCRIPTS'.lower()], help="Same as [-S --scripts], but run after calibration.") - parser.add_argument("--modules", nargs='*', metavar='module', required=False, default=HPC_DEFAULTS['MODULES'.lower()], help="Load these modules within each sbatch script.") - parser.add_argument("-w","--mpi_wrapper", metavar="path", required=False, type=str, default=HPC_DEFAULTS['MPI_WRAPPER'.lower()], - help="Use this mpi wrapper when calling threadsafe scripts [default: '{0}'].".format(HPC_DEFAULTS['MPI_WRAPPER'.lower()])) - parser.add_argument("-c","--container", metavar="path", required=False, type=str, default=HPC_DEFAULTS['CONTAINER'.lower()], help="Use this container when calling scripts [default: '{0}'].".format(HPC_DEFAULTS['CONTAINER'.lower()])) + parser.add_argument("-b","--precal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=None, help="Same as [-S --scripts], but run before calibration.") + parser.add_argument("-a","--postcal_scripts", action='append', nargs=3, metavar=('script','threadsafe','container'), required=False, type=parse_scripts, default=None, help="Same as [-S --scripts], but run after calibration.") + parser.add_argument("--modules", nargs='*', metavar='module', required=False, default=None, help="Load these modules within each sbatch script.") + parser.add_argument("-w","--mpi_wrapper", metavar="path", required=False, type=str, default=None, + help="Use this mpi wrapper when calling threadsafe scripts [default: ].") + parser.add_argument("-c","--container", metavar="path", required=False, type=str, default=None, help="Use this container when calling scripts [default:].") parser.add_argument("-n","--name", metavar="unique", required=False, type=str, default='', help="Unique name to give this pipeline run (e.g. 'run1_'), appended to the start of all job names. [default: ''].") parser.add_argument("-d","--dependencies", metavar="list", required=False, type=str, default='', help="Comma-separated list (without spaces) of SLURM job dependencies (only used when nspw=1). [default: ''].") parser.add_argument("-e","--exclude", metavar="nodes", required=False, type=str, default='', help="SLURM worker nodes to exclude [default: ''].") - parser.add_argument("-A","--account", metavar="group", required=False, type=str, default=HPC_DEFAULTS['ACCOUNTS'.lower()][0], help="SLURM accounting group to use (e.g. 'b05-pipelines-ag' - check 'sacctmgr show user $USER cluster=ilifu-slurm20 -s format=account%%30,cluster%%15') [default: 'b03-idia-ag'].") + parser.add_argument("-A","--account", metavar="group", required=False, type=str, default=None, help="SLURM accounting group to use (e.g. 'b05-pipelines-ag' - check 'sacctmgr show user $USER cluster=ilifu-slurm20 -s format=account%%30,cluster%%15') [default: 'b03-idia-ag'].") parser.add_argument("-r","--reservation", metavar="name", required=False, type=str, default='', help="SLURM reservation to use. [default: ''].") parser.add_argument("-l","--local", action="store_true", required=False, default=False, help="Build config file locally (i.e. without calling srun) [default: False].") @@ -213,6 +199,37 @@ def parse_scripts(val): args, unknown = parser.parse_known_args() + HPC_NAME = args.hpc.lower() if args.hpc in KNOWN_HPCS.keys() else "unknown" + logger.info(HPC_NAME) + HPC_DEFAULTS = KNOWN_HPCS[HPC_NAME] + # Hash table to lookup in config file + arg_hash = { + "nodes": "total_nodes_limit", + "ntasks_per_node": "ntasks_per_node_limit", + "mem": "mem_per_node_gb_limit", + "account": "accounts" + } + # Set arg values from config + for arg, val in vars(args).items(): + if val is None: + if arg in HPC_DEFAULTS.keys(): + hpc_val = HPC_DEFAULTS[arg] + logger.info(f"Setting option '{arg}' to '{hpc_val}' from HPC '{HPC_NAME}' config.") + vars(args)[arg] = hpc_val + elif arg in arg_hash.keys(): + hpc_val = HPC_DEFAULTS[arg_hash[arg]][0] if arg=="account" else HPC_DEFAULTS[arg_hash[arg]] + logger.info(f"Setting option '{arg}' to '{hpc_val}' from HPC '{HPC_NAME}' config.") + vars(args)[arg] = hpc_val + # Extract hpc name used during build and warn if not the same as CLI hpc + config_dict, config = config_parser.parse_config(args.config) + if config.has_option('run', 'hpc'): + config_hpc_name = config['run']['hpc'].strip("'") + else: + config_hpc_name = HPC_NAME + if HPC_NAME != config_hpc_name: + msg = "Entered hpc ({HPC_NAME}) is not the same as was used to build this pipeline instance! '{config_hpc_name}' was used. Pipeline may not function as expected. Please consider rebuilding pipeline with correct hpc selection." + logger.warning(msg.format(HPC_NAME=HPC_NAME, config_hpc_name=config_hpc_name)) + if len(unknown) > 0: parser.error('Unknown input argument(s) present - {0}'.format(unknown)) @@ -414,7 +431,7 @@ def write_command(script,args,mpi_wrapper,container,name='job',casa_script=False arr=($SPWs) cd ${arr[SLURM_ARRAY_TASK_ID]} - """ % SPWs.replace(',',' ').replace(SPW_PREFIX,'') + """ % SPWs.replace(',',' ').replace(HPC_DEFAULTS["SPW_PREFIX".lower()],'') command += "{mpi_wrapper} singularity exec {path_binding}{container} {plot_call} {casa_call} {script} {args}{argument_calls}".format(**params) @@ -582,7 +599,7 @@ def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,su master = open(filename,'w') master.write('#!/bin/bash\n') - SPWs = SPWs.replace(SPW_PREFIX,'') + SPWs = SPWs.replace(HPC_DEFAULTS["SPW_PREFIX".lower()],'') toplevel = len(precal_scripts + postcal_scripts) > 0 scripts = precal_scripts[:] @@ -643,7 +660,7 @@ def write_spw_master(filename,config,args,SPWs,precal_scripts,postcal_scripts,su argument_calls += " --quiet" for i,spw in enumerate(SPWs.split(',')): - master.write('echo Running pipeline in directory "{1}" for spectral window {0}{1}\n'.format(SPW_PREFIX, spw)) + master.write('echo Running pipeline in directory "{1}" for spectral window {0}{1}\n'.format(HPC_DEFAULTS["SPW_PREFIX".lower()], spw)) master.write('cd {0}\n'.format(spw)) master.write('output=$({0} --config ./{config} --run --submit --justrun {argument_calls}'.format(os.path.split(THIS_PROG)[1], config=config, argument_calls=argument_calls)) if partition: @@ -1242,7 +1259,7 @@ def format_args(config,submit,quiet,dependencies,justrun): SLURM_CONFIG_KEYS = HPC_DEFAULTS['SLURM_CONFIG_KEYS_BASE'.lower()]+HPC_DEFAULTS['SLURM_CONFIG_STR_KEYS'.lower()] kwargs = get_config_kwargs(config,'slurm',SLURM_CONFIG_KEYS) data_kwargs = get_config_kwargs(config,'data',['vis']) - get_config_kwargs(config, 'fields', HPC_DEFAULTS['FIELDS_CONFIG_KEYS'.lower()]) + field_kwargs = get_config_kwargs(config, 'fields', HPC_DEFAULTS['FIELDS_CONFIG_KEYS'.lower()]) crosscal_kwargs = get_config_kwargs(config, 'crosscal', HPC_DEFAULTS['CROSSCAL_CONFIG_KEYS'.lower()]) #Force submit=True if user has requested it during [-R --run] @@ -1473,7 +1490,7 @@ def spw_split(spw,nspw,config,mem,badfreqranges,MS,partition,createmms=True,remo #Remove SPWs entirely encompassed by bad frequency ranges (only for MHz unit) for i in range(len(lo)): - SPWs.append('{0}{1}~{2}{3}'.format(SPW_PREFIX, func(lo[i]),func(hi[i]),unit)) + SPWs.append('{0}{1}~{2}{3}'.format(HPC_DEFAULTS["SPW_PREFIX".lower()], func(lo[i]),func(hi[i]),unit)) elif ',' in spw: SPWs = spw.split(',') @@ -1492,9 +1509,9 @@ def spw_split(spw,nspw,config,mem,badfreqranges,MS,partition,createmms=True,remo low,high = get_spw_bounds(SPWs[i])[0:2] if unit == 'MHz' and remove: for freq in badfreqranges: - bad_low,bad_high = get_spw_bounds('{0}{1}'.format(SPW_PREFIX,freq))[0:2] + bad_low,bad_high = get_spw_bounds('{0}{1}'.format(HPC_DEFAULTS["SPW_PREFIX".lower()],freq))[0:2] if low >= bad_low and high <= bad_high: - logger.info("Won't process spw '{0}{1}~{2}{3}', since it's completely encompassed by bad frequency range '{3}'.".format(SPW_PREFIX,low,high,unit,freq)) + logger.info("Won't process spw '{0}{1}~{2}{3}', since it's completely encompassed by bad frequency range '{3}'.".format(HPC_DEFAULTS["SPW_PREFIX".lower()],low,high,unit,freq)) badfreq = True break if badfreq: @@ -1509,9 +1526,9 @@ def spw_split(spw,nspw,config,mem,badfreqranges,MS,partition,createmms=True,remo #Create each spw as directory and place config in there logger.info("Making {0} directories for SPWs ({1}) and copying '{2}' to each of them.".format(nspw,SPWs,config)) for spw in SPWs: - spw_config = '{0}/{1}'.format(spw.replace(SPW_PREFIX,''),config) - if not os.path.exists(spw.replace(SPW_PREFIX,'')): - os.mkdir(spw.replace(SPW_PREFIX,'')) + spw_config = '{0}/{1}'.format(spw.replace(HPC_DEFAULTS["SPW_PREFIX".lower()],''),config) + if not os.path.exists(spw.replace(HPC_DEFAULTS["SPW_PREFIX".lower()],'')): + os.mkdir(spw.replace(HPC_DEFAULTS["SPW_PREFIX".lower()],'')) copyfile(config, spw_config) config_parser.overwrite_config(spw_config, conf_dict={'spw' : "'{0}'".format(spw)}, conf_sec='crosscal') config_parser.overwrite_config(spw_config, conf_dict={'nspw' : 1}, conf_sec='crosscal') @@ -1533,7 +1550,7 @@ def spw_split(spw,nspw,config,mem,badfreqranges,MS,partition,createmms=True,remo extn = '{0}.{1}'.format(suffix[1:],extn) filebase = prefix - vis = '{0}.{1}.{2}'.format(filebase,spw.replace(SPW_PREFIX,''),extn) + vis = '{0}.{1}.{2}'.format(filebase,spw.replace(HPC_DEFAULTS["SPW_PREFIX".lower()],''),extn) logger.warning("Since script with 'partition' in its name isn't present in '{0}', assuming partition has already been done, and setting vis='{1}' in '{2}'. If '{1}' doesn't exist, please update '{2}', as the pipeline will not launch successfully.".format(config,vis,spw_config)) orig_vis = config_parser.get_key(spw_config, 'data', 'vis') config_parser.overwrite_config(spw_config, conf_dict={'orig_vis' : "'{0}'".format(orig_vis)}, conf_sec='run', sec_comment='# Internal variables for pipeline execution') From eb4aacde2823641dc3ca872af09d941c8151154e Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Sat, 2 Jul 2022 18:03:32 +1000 Subject: [PATCH 33/39] Typo --- processMeerKAT/processMeerKAT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index cbc73f9..cf27929 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -149,7 +149,7 @@ def parse_scripts(val): - parser.add_argument("--hpc",metavar='name', required=False, type=str, default="unkown", help="Name of hpc facility being used. If not known to processMeerKAT/known_hpc.cfg slurm limits are functionally removed [default: ilifu].") + parser.add_argument("--hpc",metavar='name', required=False, type=str, default="unknown", help="Name of hpc facility being used. If not known to processMeerKAT/known_hpc.cfg slurm limits are functionally removed [default: ilifu].") parser.add_argument("-M","--MS",metavar="path", required=False, type=str, help="Path to MeasurementSet.") From b13e13154edbdd425178446ae1670f2e2b356dd0 Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Fri, 8 Jul 2022 12:36:31 +1000 Subject: [PATCH 34/39] Check for HPC in config --- processMeerKAT/processMeerKAT.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/processMeerKAT/processMeerKAT.py b/processMeerKAT/processMeerKAT.py index cf27929..6b61bfb 100755 --- a/processMeerKAT/processMeerKAT.py +++ b/processMeerKAT/processMeerKAT.py @@ -141,7 +141,7 @@ def parse_scripts(val): # Begin parsing parser = argparse.ArgumentParser(prog=THIS_PROG,description='Process MeerKAT data via CASA MeasurementSet. Version: {0}'.format(__version__)) - + if os.path.isfile(known_hpc_path): KNOWN_HPCS, HPC_CONFIG = config_parser.parse_config(known_hpc_path) else: @@ -199,7 +199,16 @@ def parse_scripts(val): args, unknown = parser.parse_known_args() - HPC_NAME = args.hpc.lower() if args.hpc in KNOWN_HPCS.keys() else "unknown" + # Check for HPC in config file + config_dict, config = config_parser.parse_config(args.config) + if "run" in config_dict: + if "hpc" in config_dict["run"]: + config_hpc = config_dict["run"]["hpc"] + logger.info("Found HPC in config file: {}".format(config_hpc)) + else: + config_hpc = "unknown" + + HPC_NAME = args.hpc.lower() if args.hpc in KNOWN_HPCS.keys() and args.hpc.lower() != "unknown" else config_hpc logger.info(HPC_NAME) HPC_DEFAULTS = KNOWN_HPCS[HPC_NAME] # Hash table to lookup in config file @@ -297,8 +306,8 @@ def validate_args(args,config,parser=None): raise_error(config, msg, parser) if HPC_NAME=="unknown": - msg = "HPC facility [--hpc] is not in 'known_hpc.cfg', reverting to 'unknown' HPC. You input {0}. Pipeline will rely entirely on the specified arguemnts. No upper limits will be set. HPC specific selections within your config may cause pipeline runs to fail!" - logger.warning(msg.format(args['hpc'])) + msg = f"HPC facility is not in 'known_hpc.cfg', reverting to 'unknown' HPC. You input {0}. Pipeline will rely entirely on the specified arguemnts. No upper limits will be set. HPC specific selections within your config may cause pipeline runs to fail!" + logger.warning(msg) else: if args['ntasks_per_node'] > HPC_DEFAULTS['NTASKS_PER_NODE_LIMIT'.lower()]: From 6d088bed3d31c343cf80ea6a85c67f6379c77cdf Mon Sep 17 00:00:00 2001 From: "Thomson, Alec (CASS, Kensington)" Date: Fri, 8 Jul 2022 12:36:45 +1000 Subject: [PATCH 35/39] Update petrichor --- processMeerKAT/known_hpc.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 574a856..9adc30a 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -109,8 +109,8 @@ [petrichor] # Specify differences to ilifu - TOTAL_NODES_LIMIT = 110 - CPUS_PER_NODE_LIMIT = 64 + TOTAL_NODES_LIMIT = 110 + CPUS_PER_NODE_LIMIT = 64 NTASKS_PER_NODE_LIMIT = %(CPUS_PER_NODE_LIMIT)s MEM_PER_NODE_GB_LIMIT = 512 # GB MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1000 # @@ -119,7 +119,7 @@ PARTITION = 'defq' QOS = 'express' MPI_WRAPPER = 'mpirun' - MODULES = ['singularity','openmpi'] + MODULES = ['singularity', 'use.own', 'openmpi/2.1.1'] path_binding = '--bind /scratch1:/scratch1,/scratch2:/scratch2 ' # Must use linebreaks for #SBATCH lines! Otherwise python reads them as comments, and ignores them. submission_file_base = "#!/bin/bash{array}{exclude}{reservation}\n#SBATCH --nodes={nodes}\n#SBATCH --ntasks-per-node={tasks}\n#SBATCH --cpus-per-task={cpus}\n#SBATCH --mem={mem}GB\n#SBATCH --job-name={runname}{name}\n#SBATCH --distribution=plane={plane}\n#SBATCH --output={LOG_DIR}/%%x-{ID}.out\n#SBATCH --error={LOG_DIR}/%%x-{ID}.err\n#SBATCH --time={time}" From 25cf6ba77387533b5f35afcc836d54a39cd05c80 Mon Sep 17 00:00:00 2001 From: Srikrishna Sekhar Date: Tue, 16 Aug 2022 14:26:25 +0200 Subject: [PATCH 36/39] Updated README.md Changed casa6 --> master to reflect the merge for V2.0 --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 8533553..10f91cf 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ This pipeline is designed to run on the Ilifu cluster, making use of SLURM and M ## 1. Setup the pipeline in your environment -**Please note : These docs are for an upcoming release of the pipeline. At the time of writing (July 2022) please use the CASA6 branch of the pipeline, which is the pre-release version. The master branch will be updated at the time of release.** - In order to use the `processMeerKAT.py` script, source the `setup.sh` file, which can be done on [ilifu](https://docs.ilifu.ac.za/#/) as source /idia/software/pipelines/master/setup.sh From 6f135f01e771bc1a76b05f9d13317674825a3650 Mon Sep 17 00:00:00 2001 From: Jordan Collier Date: Wed, 17 Aug 2022 09:30:57 +1000 Subject: [PATCH 37/39] Fix broken docs link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 10f91cf..fa8aa3c 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,6 @@ Starting with v1.1 of the processMeerKAT pipeline, the default behaviour is to s 2. **Logs in the top level directory** : Logs in the top level directory (*i.e.,* the directory where the pipeline was launched) correspond to the scripts in the `precal_scripts` and `postcal_scripts` variables in the config file. These scripts are run from the top level before and after calibration respectively. By default these correspond to the scripts to calculate the reference antenna (if enabled), partition the data into SPWs, and concat the individual SPWs back into a single MS/MMS. -More detailed information about SPW splitting is found [here](/docs/processMeerKAT/config-files#spw-splitting). +More detailed information about SPW splitting is found [here](https://idia-pipelines.github.io/docs/processMeerKAT/config-files#spw-splitting). The documentation can be accessed on the [pipelines website](https://idia-pipelines.github.io/docs/processMeerKAT), or on the [Github wiki](https://github.com/idia-astro/pipelines/wiki). From fe2c8f4e515aa36761be28f085b6cff7254fb618 Mon Sep 17 00:00:00 2001 From: Jordan Collier Date: Wed, 17 Aug 2022 01:35:46 +0200 Subject: [PATCH 38/39] Delete .DS_Store --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 6ffdb830ea2134aa631de06070a55e1f24ad89d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyH3MU47J+~kytu1-Y=-sh51sY8w=`QRi#p?)G8gy9*Lpi2axz5zKW^G4u{?Mzu91w0n zW7^P`j&1(n<9X4}?dz&q?dla`<=giD_2usUwwqu7m_IyzKilvogBlhJgaV;JC=d$# zO9jxg#mXzgSVMtOAQbpgK)w$NE|?t~L;ZAM(GdVBzwv6g)>#6WBmm5gjUgg1HY(7l z>>~yn9r5IK*|9M+IxDPGBYYwzWx*CzN9{%NSyaxB`4f!d0(kyd=&s!MW>e0FRMm5#X5f$<|C Lx`c%SzoEbxg`O$a From 2f4f24894c41c8b4eafb6a303e0d88ad7e993d46 Mon Sep 17 00:00:00 2001 From: mb010 Date: Tue, 21 Feb 2023 11:52:19 +0000 Subject: [PATCH 39/39] Galahad container update --- processMeerKAT/known_hpc.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processMeerKAT/known_hpc.cfg b/processMeerKAT/known_hpc.cfg index 9adc30a..fe65ad9 100644 --- a/processMeerKAT/known_hpc.cfg +++ b/processMeerKAT/known_hpc.cfg @@ -98,7 +98,7 @@ MEM_PER_NODE_GB_LIMIT = 1500 # GB; 1.5TB available MEM_PER_NODE_GB_LIMIT_HIGHMEM = 1500 # GB; 1.5 TB availble ACCOUNTS = [''] # List of allowed accounts; Not currently used: see sbatch_file_base ### update me!!! - CONTAINER = '/share/nas/mbowles/dev/casa-6.simg' # Previously used '/share/nas/mbowles/mightee/casa-stable.simg' + CONTAINER = '/share/nas/mbowles/casa-6_v2.simg' # Previously used '/share/nas/mbowles/mightee/casa-stable.simg' PARTITION = 'CLUSTER' QOS = 'Normal' MPI_WRAPPER = 'mpirun --mca oob tcp'