From f23d8049cdcc203d237241642d5e379b252772b4 Mon Sep 17 00:00:00 2001 From: Qing Liu Date: Tue, 23 Jun 2026 11:55:00 -0400 Subject: [PATCH 1/7] updated setup files and templates for LADAS --- GEOSldas_App/GEOSldas_HISTdet.rc | 2 +- .../LDASsa_SPECIAL_inputs_ensupd.nml | 16 ++++++++-------- GEOSldas_App/ldas_setup | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/GEOSldas_App/GEOSldas_HISTdet.rc b/GEOSldas_App/GEOSldas_HISTdet.rc index 55995fc3..dde3b326 100644 --- a/GEOSldas_App/GEOSldas_HISTdet.rc +++ b/GEOSldas_App/GEOSldas_HISTdet.rc @@ -73,7 +73,7 @@ catch_progn_incr.fields: 'TCFSAT_INCR' , 'LANDASSIM' , tavg3_1d_lnd_Nt.template: '%y4%m2%d2_%h2%n2z.bin', tavg3_1d_lnd_Nt.mode: 'time-averaged', tavg3_1d_lnd_Nt.frequency: 030000, - tavg3_1d_lnd_Nt.ref_time: 013000, + tavg3_1d_lnd_Nt.ref_time: 000000, tavg3_1d_lnd_Nt.fields: 'WET3' , 'ENSAVG' , 'GWETPROF' , 'WET2' , 'ENSAVG' , 'GWETROOT' , 'WET1' , 'ENSAVG' , 'GWETTOP' , diff --git a/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml b/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml index e8ecb12e..bcd27ec9 100644 --- a/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml +++ b/GEOSldas_App/LandAtmDAS_nml/LDASsa_SPECIAL_inputs_ensupd.nml @@ -269,8 +269,8 @@ obs_param_nml(31)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(31)%name = '' obs_param_nml(31)%maskpath = '' obs_param_nml(31)%maskname = '' -obs_param_nml(31)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(31)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(31)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(31)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(31)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(31)%flistname = 'SMAP_L1C_TB_A_list.txt' obs_param_nml(31)%errstd = 4. @@ -307,8 +307,8 @@ obs_param_nml(32)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(32)%name = '' obs_param_nml(32)%maskpath = '' obs_param_nml(32)%maskname = '' -obs_param_nml(32)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(32)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(32)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(32)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(32)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(32)%flistname = 'SMAP_L1C_TB_D_list.txt' obs_param_nml(32)%errstd = 4. @@ -345,8 +345,8 @@ obs_param_nml(33)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(33)%name = '' obs_param_nml(33)%maskpath = '' obs_param_nml(33)%maskname = '' -obs_param_nml(33)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(33)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(33)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(33)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(33)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(33)%flistname = 'SMAP_L1C_TB_A_list.txt' obs_param_nml(33)%errstd = 4. @@ -383,8 +383,8 @@ obs_param_nml(34)%path = '/discover/nobackup/projects/gmao/smap/SMAP_L obs_param_nml(34)%name = '' obs_param_nml(34)%maskpath = '' obs_param_nml(34)%maskname = '' -obs_param_nml(34)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v000/x6C_GLOBAL/' -obs_param_nml(34)%scalename = 'ScMO_SMAP__e24_zscore_stats_2015_p19_2020_p18_hscale_0.00_W_9p_Nmin_20' +obs_param_nml(34)%scalepath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/scaling/SMAP_L1C_Tb_pentad/LADAS_v002/x6C_GLOBAL/' +obs_param_nml(34)%scalename = 'ScMO_SMAP__e06_zscore_stats' obs_param_nml(34)%flistpath = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/obs/SMAP/L1C_TB_flist/SPL4SM_OL8000/' obs_param_nml(34)%flistname = 'SMAP_L1C_TB_D_list.txt' obs_param_nml(34)%errstd = 4. diff --git a/GEOSldas_App/ldas_setup b/GEOSldas_App/ldas_setup index 845312b6..f4028925 100755 --- a/GEOSldas_App/ldas_setup +++ b/GEOSldas_App/ldas_setup @@ -239,7 +239,7 @@ class LDASsetup: self.rqdExeInp['RESTART_DOMAIN'] = self.agcm_res +'x6C_GLOBAL' # the following are not in default rqdExeInp list; hardwire for now - self.rqdExeInp['MWRTM_PATH'] = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/RTM_params/RTMParam_SMAP_L4SM_v006/' + self.rqdExeInp['MWRTM_PATH'] = '/discover/nobackup/projects/gmao/smap/LDAS_inputs_for_LADAS/RTM_params/RTMParam_SMAP_L4SM_v007/' self.rqdExeInp['LAND_ASSIM'] = "YES" self.rqdExeInp['MET_HINTERP'] = 0 self.landassim_dt = 10800 # seconds @@ -612,7 +612,7 @@ class LDASsetup: else : self.rqdRmInp['account'] = cmdLineArgs['account'] self.rqdRmInp['walltime'] = "01:00:00" - self.rqdRmInp['ntasks_model'] = 120 + self.rqdRmInp['ntasks_model'] = 138 # print rm inputs From 6900dda2e222986ae76a5f2130237ff72fd766d6 Mon Sep 17 00:00:00 2001 From: Qing Liu Date: Tue, 23 Jun 2026 16:40:15 -0400 Subject: [PATCH 2/7] add lines in lenkf.j to remove unwanted files --- GEOSldas_App/lenkf_j_template.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/GEOSldas_App/lenkf_j_template.py b/GEOSldas_App/lenkf_j_template.py index 0ed56fd0..97afaae9 100644 --- a/GEOSldas_App/lenkf_j_template.py +++ b/GEOSldas_App/lenkf_j_template.py @@ -408,6 +408,15 @@ /bin/cp LDAS.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_LDAS_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt /bin/cp CAP.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_CAP_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt + # Remove existing *.nml, *obsparam*, and *lmc* files before each lenkf.j + # submission in LADAS coupled runs. Each run generates these files, and + # pre-existing ones will cause LDAS to crash. + if ( $LADAS_COUPLING > 0 ) then + /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_smapL4SMlmc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.bin + /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_*_inputs.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.nml + /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_obsparam.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt + endif + # Run GEOSldas.x # -------------- # clean up From b836898921f98c49f21944d821b387bfcb7f2e49 Mon Sep 17 00:00:00 2001 From: Rolf Reichle <54944691+gmao-rreichle@users.noreply.github.com> Date: Tue, 23 Jun 2026 17:27:03 -0400 Subject: [PATCH 3/7] Clarified comment from previous commit --- GEOSldas_App/lenkf_j_template.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/GEOSldas_App/lenkf_j_template.py b/GEOSldas_App/lenkf_j_template.py index 97afaae9..8f0804a8 100644 --- a/GEOSldas_App/lenkf_j_template.py +++ b/GEOSldas_App/lenkf_j_template.py @@ -404,13 +404,13 @@ set old_catch_param = `/usr/bin/readlink -f $old_catch_param` endif - /bin/cp LDAS.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_LDAS_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt - /bin/cp CAP.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_CAP_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt + /bin/cp CAP.rc $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_CAP_rc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.txt + + # For LADAS coupled runs, remove existing *.nml, *obsparam*, and *lmc* files to avoid GEOSldas.x crashing after rewind. + # (These files are created at the beginning of each GEOSldas run. To avoid overwriting existing simulations, + # GEOSldas.x intentionally stops when these files are present.) - # Remove existing *.nml, *obsparam*, and *lmc* files before each lenkf.j - # submission in LADAS coupled runs. Each run generates these files, and - # pre-existing ones will cause LDAS to crash. if ( $LADAS_COUPLING > 0 ) then /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_smapL4SMlmc.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.bin /bin/rm $EXPDIR/output/$EXPDOMAIN/rc_out/Y${{bYEAR}}/M${{bMON}}/${{EXPID}}.ldas_*_inputs.${{bYEAR}}${{bMON}}${{bDAY}}_${{bHour}}${{bMin}}z.nml From 495e8b3ee3ffc12e69615e90597756f2e5b88b2e Mon Sep 17 00:00:00 2001 From: Rolf Reichle <54944691+gmao-rreichle@users.noreply.github.com> Date: Tue, 23 Jun 2026 17:33:27 -0400 Subject: [PATCH 4/7] Updated CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69f0ea67..a28aaa12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Miscellaneous updates and fixes for land-atm DAS. + ### Fixed ### Removed From 8389c96658ccf466a738811bbd2d57c79a24f91c Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 8 Jul 2026 15:19:34 -0400 Subject: [PATCH 5/7] add ntasks-per-node --- GEOSldas_App/ldas_setup | 15 +++++++++++++-- GEOSldas_App/lenkf_j_template.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/GEOSldas_App/ldas_setup b/GEOSldas_App/ldas_setup index f4028925..7e9469af 100755 --- a/GEOSldas_App/ldas_setup +++ b/GEOSldas_App/ldas_setup @@ -63,7 +63,7 @@ class LDASsetup: # ------ # Optional resource manager input fields # ------ - optSlurmInpKeys = ['job_name', 'qos', 'oserver_nodes', 'writers-per-node', 'constraint'] + optSlurmInpKeys = ['job_name', 'qos', 'oserver_nodes', 'writers-per-node', 'constraint', 'ntasks-per-node'] # =============================================================================================== @@ -613,7 +613,7 @@ class LDASsetup: self.rqdRmInp['account'] = cmdLineArgs['account'] self.rqdRmInp['walltime'] = "01:00:00" self.rqdRmInp['ntasks_model'] = 138 - + self.optRmInp['ntasks-per-node'] = cmdLineArgs['ntasks-per-node'] # print rm inputs if self.verbose: @@ -1587,6 +1587,10 @@ class LDASsetup: if 'constraint' in self.optRmInp: constraint = self.optRmInp['constraint'] + MY_NTASKS_PER_NODE='' + if 'ntasks-per-node' in self.optRmInp: + MY_NTASKS_PER_NODE='--ntasks-per-node='+self.optRmInp['ntasks-per-node'] + SBATCHQSUB = 'sbatch' if self.GEOS_SITE == 'NAS': SBATCHQSUB = 'qsub' @@ -1599,6 +1603,7 @@ class LDASsetup: MY_ACCOUNT = self.rqdRmInp['account'], MY_WALLTIME = self.rqdRmInp['walltime'], MY_NTASKS_MODEL = str(self.rqdRmInp['ntasks_model']), + MY_NTASKS_PER_NODE = MY_NTASKS_PER_NODE MY_CONSTRAINT = constraint, MY_OSERVER_NODES = str(self.optRmInp['oserver_nodes']), MY_WRITERS_NPES = str(self.optRmInp['writers-per-node']), @@ -1872,6 +1877,7 @@ def _printRmInputKeys(rqdRmInpKeys, optRmInpKeys): print ('# IMPORTANT REQUIREMENT: total #writers = writers-per-node * oserver_nodes >= 2') print ('# Jobs will hang when oserver_nodes = writers-per-node = 1.') print ('# - constraint = name of chip set(s) (NCCS default is "[mil|cas]", NAS default is "cas_ait").') + print ('# - ntasks-per-node = number of processes for each computing node.') print ('#') for key in optRmInpKeys: print ('#'+key + ':') @@ -1989,6 +1995,11 @@ def parseCmdLine(): '--nens', help='number of ensemble members. Required when exeinp is dummy', type=str + ) + p_setup.add_argument( + '--ntasks-per-node', + help='number of processes for each computing node. Required when exeinp is dummy', + type=str ) # obsolete command line args diff --git a/GEOSldas_App/lenkf_j_template.py b/GEOSldas_App/lenkf_j_template.py index 8f0804a8..466a5a10 100644 --- a/GEOSldas_App/lenkf_j_template.py +++ b/GEOSldas_App/lenkf_j_template.py @@ -12,7 +12,7 @@ #SBATCH --error={MY_EXPDIR}/scratch/GEOSldas_err_txt #SBATCH --account={MY_ACCOUNT} #SBATCH --time={MY_WALLTIME} -#SBATCH --ntasks={MY_NTASKS_MODEL} +#SBATCH --ntasks={MY_NTASKS_MODEL} {MY_NTASKS_PER_NODE} #SBATCH --job-name={MY_JOB} #SBATCH --qos={MY_QOS} #SBATCH --constraint={MY_CONSTRAINT} From f4350c3054cf17345fddcaf30081b4c842e3a069 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 8 Jul 2026 15:40:09 -0400 Subject: [PATCH 6/7] hard coded ntasks-per-node for adas --- GEOSldas_App/ldas_setup | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/GEOSldas_App/ldas_setup b/GEOSldas_App/ldas_setup index 7e9469af..8d90e947 100755 --- a/GEOSldas_App/ldas_setup +++ b/GEOSldas_App/ldas_setup @@ -612,8 +612,8 @@ class LDASsetup: else : self.rqdRmInp['account'] = cmdLineArgs['account'] self.rqdRmInp['walltime'] = "01:00:00" - self.rqdRmInp['ntasks_model'] = 138 - self.optRmInp['ntasks-per-node'] = cmdLineArgs['ntasks-per-node'] + self.rqdRmInp['ntasks_model'] = 120 + self.optRmInp['ntasks-per-node'] = 30 # print rm inputs if self.verbose: @@ -1589,7 +1589,7 @@ class LDASsetup: MY_NTASKS_PER_NODE='' if 'ntasks-per-node' in self.optRmInp: - MY_NTASKS_PER_NODE='--ntasks-per-node='+self.optRmInp['ntasks-per-node'] + MY_NTASKS_PER_NODE='--ntasks-per-node='+str(self.optRmInp['ntasks-per-node']) SBATCHQSUB = 'sbatch' if self.GEOS_SITE == 'NAS': @@ -1996,11 +1996,6 @@ def parseCmdLine(): help='number of ensemble members. Required when exeinp is dummy', type=str ) - p_setup.add_argument( - '--ntasks-per-node', - help='number of processes for each computing node. Required when exeinp is dummy', - type=str - ) # obsolete command line args p_setup.add_argument( From a4dd4b5167573b283124d05bd216cc2e2dfcdfd3 Mon Sep 17 00:00:00 2001 From: Qing Liu Date: Thu, 9 Jul 2026 12:33:32 -0400 Subject: [PATCH 7/7] add missing comma. --- GEOSldas_App/ldas_setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GEOSldas_App/ldas_setup b/GEOSldas_App/ldas_setup index 8d90e947..9da751a2 100755 --- a/GEOSldas_App/ldas_setup +++ b/GEOSldas_App/ldas_setup @@ -1603,7 +1603,7 @@ class LDASsetup: MY_ACCOUNT = self.rqdRmInp['account'], MY_WALLTIME = self.rqdRmInp['walltime'], MY_NTASKS_MODEL = str(self.rqdRmInp['ntasks_model']), - MY_NTASKS_PER_NODE = MY_NTASKS_PER_NODE + MY_NTASKS_PER_NODE = MY_NTASKS_PER_NODE, MY_CONSTRAINT = constraint, MY_OSERVER_NODES = str(self.optRmInp['oserver_nodes']), MY_WRITERS_NPES = str(self.optRmInp['writers-per-node']),