diff --git a/CHANGELOG.md b/CHANGELOG.md index 69f0ea67..a28aaa12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Miscellaneous updates and fixes for land-atm DAS. + ### Fixed ### Removed diff --git a/GEOSldas_App/GEOSldas_HISTdet.rc b/GEOSldas_App/GEOSldas_HISTdet.rc index 55995fc3..dde3b326 100644 --- a/GEOSldas_App/GEOSldas_HISTdet.rc +++ b/GEOSldas_App/GEOSldas_HISTdet.rc @@ -73,7 +73,7 @@ catch_progn_incr.fields: 'TCFSAT_INCR' , 'LANDASSIM' , tavg3_1d_lnd_Nt.template: '%y4%m2%d2_%h2%n2z.bin', tavg3_1d_lnd_Nt.mode: 'time-averaged', tavg3_1d_lnd_Nt.frequency: 030000, - tavg3_1d_lnd_Nt.ref_time: 013000, + tavg3_1d_lnd_Nt.ref_time: 000000, tavg3_1d_lnd_Nt.fields: 'WET3' , 'ENSAVG' , 'GWETPROF' , 'WET2' , 'ENSAVG' , 'GWETROOT' , 'WET1' , 'ENSAVG' , 'GWETTOP' , diff --git a/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml b/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml index e8ecb12e..bcd27ec9 100644 --- a/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml +++ b/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml @@ -269,8 +269,8 @@ obs_param_nml(31)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(31)%name = '' obs_param_nml(31)%maskpath = '' obs_param_nml(31)%maskname = '' -obs_param_nml(31)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(31)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(31)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(31)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(31)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(31)%flistname = 'SMAP_L1C_TB_A_list.txt' obs_param_nml(31)%errstd = 4. @@ -307,8 +307,8 @@ obs_param_nml(32)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(32)%name = '' obs_param_nml(32)%maskpath = '' obs_param_nml(32)%maskname = '' -obs_param_nml(32)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(32)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(32)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(32)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(32)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(32)%flistname = 'SMAP_L1C_TB_D_list.txt' obs_param_nml(32)%errstd = 4. @@ -345,8 +345,8 @@ obs_param_nml(33)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(33)%name = '' obs_param_nml(33)%maskpath = '' obs_param_nml(33)%maskname = '' -obs_param_nml(33)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(33)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(33)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(33)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(33)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(33)%flistname = 'SMAP_L1C_TB_A_list.txt' obs_param_nml(33)%errstd = 4. @@ -383,8 +383,8 @@ obs_param_nml(34)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(34)%name = '' obs_param_nml(34)%maskpath = '' obs_param_nml(34)%maskname = '' -obs_param_nml(34)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(34)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(34)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(34)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(34)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(34)%flistname = 'SMAP_L1C_TB_D_list.txt' obs_param_nml(34)%errstd = 4. diff --git a/GEOSldas_App/ldas_setup b/GEOSldas_App/ldas_setup index 845312b6..9da751a2 100755 --- a/GEOSldas_App/ldas_setup +++ b/GEOSldas_App/ldas_setup @@ -63,7 +63,7 @@ class LDASsetup: # ------ # Optional resource manager input fields # ------ - optSlurmInpKeys = ['job_name', 'qos', 'oserver_nodes', 'writers-per-node', 'constraint'] + optSlurmInpKeys = ['job_name', 'qos', 'oserver_nodes', 'writers-per-node', 'constraint', 'ntasks-per-node'] # =============================================================================================== @@ -239,7 +239,7 @@ class LDASsetup: self.rqdExeInp['RESTART_DOMAIN'] = self.agcm_res +'x6C_GLOBAL' # the following are not in default rqdExeInp list; hardwire for now - self.rqdExeInp['MWRTM_PATH'] = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/RTM_params/RTMParam_SMAP_L4SM_v006/' + self.rqdExeInp['MWRTM_PATH'] = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/RTM_params/RTMParam_SMAP_L4SM_v007/' self.rqdExeInp['LAND_ASSIM'] = "YES" self.rqdExeInp['MET_HINTERP'] = 0 self.landassim_dt = 10800 # seconds @@ -613,7 +613,7 @@ class LDASsetup: self.rqdRmInp['account'] = cmdLineArgs['account'] self.rqdRmInp['walltime'] = "01:00:00" self.rqdRmInp['ntasks_model'] = 120 - + self.optRmInp['ntasks-per-node'] = 30 # print rm inputs if self.verbose: @@ -1587,6 +1587,10 @@ class LDASsetup: if 'constraint' in self.optRmInp: constraint = self.optRmInp['constraint'] + MY_NTASKS_PER_NODE='' + if 'ntasks-per-node' in self.optRmInp: + MY_NTASKS_PER_NODE='--ntasks-per-node='+str(self.optRmInp['ntasks-per-node']) + SBATCHQSUB = 'sbatch' if self.GEOS_SITE == 'NAS': SBATCHQSUB = 'qsub' @@ -1599,6 +1603,7 @@ class LDASsetup: MY_ACCOUNT = self.rqdRmInp['account'], MY_WALLTIME = self.rqdRmInp['walltime'], MY_NTASKS_MODEL = str(self.rqdRmInp['ntasks_model']), + MY_NTASKS_PER_NODE = MY_NTASKS_PER_NODE, MY_CONSTRAINT = constraint, MY_OSERVER_NODES = str(self.optRmInp['oserver_nodes']), MY_WRITERS_NPES = str(self.optRmInp['writers-per-node']), @@ -1872,6 +1877,7 @@ def _printRmInputKeys(rqdRmInpKeys, optRmInpKeys): print ('# IMPORTANT REQUIREMENT: total #writers = writers-per-node * oserver_nodes >= 2') print ('# Jobs will hang when oserver_nodes = writers-per-node = 1.') print ('# - constraint = name of chip set(s) (NCCS default is "[mil|cas]", NAS default is "cas_ait").') + print ('# - ntasks-per-node = number of processes for each computing node.') print ('#') for key in optRmInpKeys: print ('#'+key + ':') @@ -1989,7 +1995,7 @@ def parseCmdLine(): '--nens', help='number of ensemble members. Required when exeinp is dummy', type=str - ) + ) # obsolete command line args p_setup.add_argument( diff --git a/GEOSldas_App/lenkf_j_template.py b/GEOSldas_App/lenkf_j_template.py index 0ed56fd0..466a5a10 100644 --- a/GEOSldas_App/lenkf_j_template.py +++ b/GEOSldas_App/lenkf_j_template.py @@ -12,7 +12,7 @@ #SBATCH --error={MY_EXPDIR}/scratch/GEOSldas_err_txt #SBATCH --account={MY_ACCOUNT} #SBATCH --time={MY_WALLTIME} -#SBATCH --ntasks={MY_NTASKS_MODEL} +#SBATCH --ntasks={MY_NTASKS_MODEL} {MY_NTASKS_PER_NODE} #SBATCH --job-name={MY_JOB} #SBATCH --qos={MY_QOS} #SBATCH --constraint={MY_CONSTRAINT} @@ -404,10 +404,19 @@ set old_catch_param = `/usr/bin/readlink -f $old_catch_param` endif - /bin/cp LDAS.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_LDAS_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt - /bin/cp CAP.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_CAP_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt + /bin/cp CAP.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_CAP_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt + + # For LADAS coupled runs, remove existing *.nml, *obsparam*, and *lmc* files to avoid GEOSldas.x crashing after rewind. + # (These files are created at the beginning of each GEOSldas run. To avoid overwriting existing simulations, + # GEOSldas.x intentionally stops when these files are present.) + if ( $LADAS_COUPLING > 0 ) then + /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_smapL4SMlmc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.bin + /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_*_inputs.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.nml + /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_obsparam.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt + endif + # Run GEOSldas.x # -------------- # clean up