From fff056489181d2d4943fb76c34a6de1eae12e300 Mon Sep 17 00:00:00 2001 From: Swapan Mallick Date: Tue, 23 Jan 2024 09:41:37 +0000 Subject: [PATCH 01/20] Festat standalone is now added --- scripts/RUN_FESTAT.job | 99 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 scripts/RUN_FESTAT.job diff --git a/scripts/RUN_FESTAT.job b/scripts/RUN_FESTAT.job new file mode 100644 index 0000000..c7b4386 --- /dev/null +++ b/scripts/RUN_FESTAT.job @@ -0,0 +1,99 @@ +#!/bin/bash +#SBATCH --cpus-per-task=8 +#SBATCH --error=festat_OUT.log +#SBATCH --job-name=FESTAT +#SBATCH --nodes=37 +#SBATCH --output=festat_OUT.log +#SBATCH --qos=np +#SBATCH --ntasks-per-node=27 +#SBATCH --time=02:00:00 +# +# +# Magnus Lindskog 20211227 based on Javier Sanchez original version. +# Jan Barkmeijer: adapted to use CY46 FESTAT +# NPROC the total number of forecasts and +# NFRGRP to the number of forecasts per cycle +# so you may want to change the loop over the forecasts +# Swapan Mallick 20231122 adapted to 10-member EPS CY46 FESTAT_CY46 and FA file system +# Swapan Mallick 20231210 FESTAT using 2500 grid point 10 ensemble FA file ; 999 FA file +# +# +export MPPEXEC="srun" +export NPROC=999 +export NPROCX=1 +export NPROCY=1 +export OMP_MAX_ACTIVE_LEVELS=1 +export OMP_NUM_THREADS=8 +export OMP_STACKSIZE=256M +eojcmd="" +ulimit -S -s unlimited || ulimit -s +ulimit -S -m unlimited || ulimit -m +ulimit -S -d unlimited || ulimit -d +set -e # stop the shell on first error +## +## +EXP=CARRA2_CY46 +# +VLEV=65 +# adapt to relevant input +HM_DATA=/ec/res4/scratch/swe4281/hm_home/$EXP +BINDIR=/ec/res4/scratch/swe4281/hm_home/$EXP/bin/R64 +BINDIR2=/perm/swe4281/hm_lib/$EXP +DATADIR=/scratch/swe4281/FESTAT_WORK/EXPERIMENT_ALL +# +source $BINDIR2/Env_system +echo $BINDIR/FESTAT +echo $BINDIR2/Env_system +## +## +DTGBEG=2022010300 +DTGEND=2022013000 +## +MM=`echo $DTGBEG | cut -c5-6` +YY=`echo $DTGBEG | cut -c1-4` + +fctime=6 # Change here for forecast 3HR or 6HR forecast file + +jforecast=1 # start of indexing + +# First five members from a summer month +date=$DTGBEG +while [ $date -le $DTGEND ] +do + DD=$( $BINDIR2/scr/mandtg -day $date ) + HH=$( $BINDIR2/scr/mandtg -hour $date ) + + for i in 1 2 3 4 5 6 7 8 9 + do + index=$(printf "%03d" $jforecast) + ln -sf $DATADIR/$EXP/$YY/$MM/$DD/$HH/mbr00$i/ICMSHHARM+000$fctime ICMSHHARM+0$index + echo $DATADIR/$EXP/$YY/$MM/$DD/$HH/mbr00$i/ICMSHHARM+000$fctime, ICMSHHARM+0$index + jforecast=$( expr $jforecast + 1 ) + done + date=$( $BINDIR2/scr/mandtg $date + 6 ) +done +# Get the domain properties +# Build the namelist and run festat +cat > fort.4 << EOF +&NAMFESTAT + NCASES=$NPROC, + LSTABAL=.true., + LANAFBAL=.true., + LELAM=.TRUE., + CNAME='ICMSHHARM+0', + NFRGRP=9, + NPRINTLEV=0, + LOZONE=.FALSE., + LOZBAL=.FALSE., + LUVBAL=.FALSE., + OUTBAL='stab.bal' + OUTCVT='stab.cvt', + OUTCVU='stab.cv', +/ +EOF + +$MPPEXEC $BINDIR/FESTAT || exit +# +rm -rf ICMSHHARM+0* +trap - 0 +exit From 364c199214876bc2fc7ccfb44bd2aac6cec0130f Mon Sep 17 00:00:00 2001 From: Swapan Mallick Date: Tue, 23 Jan 2024 09:48:35 +0000 Subject: [PATCH 02/20] name consistancy --- scripts/{RUN_FESTAT.job => festat.job} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{RUN_FESTAT.job => festat.job} (100%) diff --git a/scripts/RUN_FESTAT.job b/scripts/festat.job similarity index 100% rename from scripts/RUN_FESTAT.job rename to scripts/festat.job From 77a283b1decc115719e077d6916acb88a36137c5 Mon Sep 17 00:00:00 2001 From: Swapan Mallick Date: Fri, 26 Jan 2024 14:00:21 +0000 Subject: [PATCH 03/20] festat script Updated and split with two --- scripts/festat_S1.py | 48 ++++++++++++++++++++++++++++++++++++++++++ scripts/festat_S2.job | 49 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 scripts/festat_S1.py create mode 100644 scripts/festat_S2.job diff --git a/scripts/festat_S1.py b/scripts/festat_S1.py new file mode 100644 index 0000000..02215e0 --- /dev/null +++ b/scripts/festat_S1.py @@ -0,0 +1,48 @@ +# +import os +from datetime import datetime, timedelta + +#---------INPUT NEEDED----------- +EXP = "SURF_CARRA2_CY46" # Change the experiment name here +dtgbeg = datetime(2022, 1, 15, 00) # Change Start date +dtgend = datetime(2022, 2, 12, 18) # Change End date +#-------------------------------- +# +cycl = timedelta(hours=6) # Assimilation cycle at 00 06 12 and 18 UTC +SCRATCH = os.environ['SCRATCH'] +HM_DATA = os.path.join(SCRATCH,'hm_home',EXP,'archive') +TMPOUT = os.path.join(SCRATCH,'tmp_festat') # Working directory 'tmp_festat' +os.makedirs(TMPOUT, mode=0o755, exist_ok=True) +os.chdir(TMPOUT) +ens_mem=range(1,10) # Ensemble member (ens1,ens2.....,ens9) +filename = "ICMSHHARM+0006" # 6 hour forecast FA file +fnumber=1 +# +while dtgbeg <= dtgend: + YY=dtgbeg.isoformat()[0:4] + MM=dtgbeg.isoformat()[5:7] + DD=dtgbeg.isoformat()[8:10] + HH=dtgbeg.isoformat()[11:13] + for mm in ens_mem: + mem='mbr00'.ljust(6,str(mm)) + target = os.path.join(HM_DATA, YY, MM, DD, HH, mem, filename) + print(target) + # + if ( fnumber <= 9 ): + filelink = filename[:-1]+str(fnumber) + elif ( fnumber >= 10 and fnumber <= 99 ): + filelink = filename[:-2]+str(fnumber) + elif ( fnumber >= 100 and fnumber <= 999 ): + filelink = filename[:-3]+str(fnumber) + else: + break + # + if os.path.isfile(filelink): + os.remove(filelink) + os.symlink(target ,filelink) + fnumber += 1 + + dtgbeg += cycl +# +print(TMPOUT) +quit() diff --git a/scripts/festat_S2.job b/scripts/festat_S2.job new file mode 100644 index 0000000..d5b24cf --- /dev/null +++ b/scripts/festat_S2.job @@ -0,0 +1,49 @@ +#!/bin/bash +#SBATCH --cpus-per-task=8 +#SBATCH --error=festat.log +#SBATCH --job-name=FESTAT +#SBATCH --nodes=37 +#SBATCH --output=festat.log +#SBATCH --qos=np +#SBATCH --ntasks-per-node=27 +#SBATCH --time=02:00:00 +# +# +export MPPEXEC="srun" +export NPROC=999 +export NPROCX=1 +export NPROCY=1 +export OMP_MAX_ACTIVE_LEVELS=1 +export OMP_NUM_THREADS=8 +export OMP_STACKSIZE=256M +eojcmd="" +ulimit -S -s unlimited || ulimit -s +ulimit -S -m unlimited || ulimit -m +ulimit -S -d unlimited || ulimit -d +set -e +## +EXP=SURF_CARRA2_CY46 +ENV=${HOME}/hm_home/$EXP +# Build the namelist +cat > fort.4 << EOF +&NAMFESTAT + NCASES=$NPROC, + LSTABAL=.true., + LANAFBAL=.true., + LELAM=.TRUE., + CNAME='ICMSHHARM+0', + NFRGRP=9, + NPRINTLEV=0, + LOZONE=.FALSE., + LOZBAL=.FALSE., + LUVBAL=.FALSE., + OUTBAL='stab.bal' + OUTCVT='stab.cvt', + OUTCVU='stab.cv', +/ +EOF +source ${ENV}/Env_system +srun FESTAT || exit +# +trap - 0 +exit From 345c374d46feb2fb79891037051de981ac99709e Mon Sep 17 00:00:00 2001 From: Swapan Mallick Date: Thu, 8 Feb 2024 11:13:51 +0000 Subject: [PATCH 04/20] Festat script updated --- scripts/festat.job | 99 ------------------------------------------- scripts/festat_S1.py | 51 +++++++++------------- scripts/festat_S2.job | 0 3 files changed, 21 insertions(+), 129 deletions(-) delete mode 100644 scripts/festat.job mode change 100644 => 100755 scripts/festat_S1.py mode change 100644 => 100755 scripts/festat_S2.job diff --git a/scripts/festat.job b/scripts/festat.job deleted file mode 100644 index c7b4386..0000000 --- a/scripts/festat.job +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -#SBATCH --cpus-per-task=8 -#SBATCH --error=festat_OUT.log -#SBATCH --job-name=FESTAT -#SBATCH --nodes=37 -#SBATCH --output=festat_OUT.log -#SBATCH --qos=np -#SBATCH --ntasks-per-node=27 -#SBATCH --time=02:00:00 -# -# -# Magnus Lindskog 20211227 based on Javier Sanchez original version. -# Jan Barkmeijer: adapted to use CY46 FESTAT -# NPROC the total number of forecasts and -# NFRGRP to the number of forecasts per cycle -# so you may want to change the loop over the forecasts -# Swapan Mallick 20231122 adapted to 10-member EPS CY46 FESTAT_CY46 and FA file system -# Swapan Mallick 20231210 FESTAT using 2500 grid point 10 ensemble FA file ; 999 FA file -# -# -export MPPEXEC="srun" -export NPROC=999 -export NPROCX=1 -export NPROCY=1 -export OMP_MAX_ACTIVE_LEVELS=1 -export OMP_NUM_THREADS=8 -export OMP_STACKSIZE=256M -eojcmd="" -ulimit -S -s unlimited || ulimit -s -ulimit -S -m unlimited || ulimit -m -ulimit -S -d unlimited || ulimit -d -set -e # stop the shell on first error -## -## -EXP=CARRA2_CY46 -# -VLEV=65 -# adapt to relevant input -HM_DATA=/ec/res4/scratch/swe4281/hm_home/$EXP -BINDIR=/ec/res4/scratch/swe4281/hm_home/$EXP/bin/R64 -BINDIR2=/perm/swe4281/hm_lib/$EXP -DATADIR=/scratch/swe4281/FESTAT_WORK/EXPERIMENT_ALL -# -source $BINDIR2/Env_system -echo $BINDIR/FESTAT -echo $BINDIR2/Env_system -## -## -DTGBEG=2022010300 -DTGEND=2022013000 -## -MM=`echo $DTGBEG | cut -c5-6` -YY=`echo $DTGBEG | cut -c1-4` - -fctime=6 # Change here for forecast 3HR or 6HR forecast file - -jforecast=1 # start of indexing - -# First five members from a summer month -date=$DTGBEG -while [ $date -le $DTGEND ] -do - DD=$( $BINDIR2/scr/mandtg -day $date ) - HH=$( $BINDIR2/scr/mandtg -hour $date ) - - for i in 1 2 3 4 5 6 7 8 9 - do - index=$(printf "%03d" $jforecast) - ln -sf $DATADIR/$EXP/$YY/$MM/$DD/$HH/mbr00$i/ICMSHHARM+000$fctime ICMSHHARM+0$index - echo $DATADIR/$EXP/$YY/$MM/$DD/$HH/mbr00$i/ICMSHHARM+000$fctime, ICMSHHARM+0$index - jforecast=$( expr $jforecast + 1 ) - done - date=$( $BINDIR2/scr/mandtg $date + 6 ) -done -# Get the domain properties -# Build the namelist and run festat -cat > fort.4 << EOF -&NAMFESTAT - NCASES=$NPROC, - LSTABAL=.true., - LANAFBAL=.true., - LELAM=.TRUE., - CNAME='ICMSHHARM+0', - NFRGRP=9, - NPRINTLEV=0, - LOZONE=.FALSE., - LOZBAL=.FALSE., - LUVBAL=.FALSE., - OUTBAL='stab.bal' - OUTCVT='stab.cvt', - OUTCVU='stab.cv', -/ -EOF - -$MPPEXEC $BINDIR/FESTAT || exit -# -rm -rf ICMSHHARM+0* -trap - 0 -exit diff --git a/scripts/festat_S1.py b/scripts/festat_S1.py old mode 100644 new mode 100755 index 02215e0..883ba45 --- a/scripts/festat_S1.py +++ b/scripts/festat_S1.py @@ -2,47 +2,38 @@ import os from datetime import datetime, timedelta -#---------INPUT NEEDED----------- -EXP = "SURF_CARRA2_CY46" # Change the experiment name here -dtgbeg = datetime(2022, 1, 15, 00) # Change Start date -dtgend = datetime(2022, 2, 12, 18) # Change End date +#---------INPUT STAFF----------- +EXP = "SURF_CARRA2_CY46" # Change the experiment name here +dtg = datetime.strptime('2022011500', '%Y%m%d%H') # Change Start date +dtgend = datetime.strptime('2022021218', '%Y%m%d%H') # Change End date #-------------------------------- -# -cycl = timedelta(hours=6) # Assimilation cycle at 00 06 12 and 18 UTC +cycl = timedelta(hours=6) # Assimilation cycle at 00 06 12 and 18 UTC SCRATCH = os.environ['SCRATCH'] HM_DATA = os.path.join(SCRATCH,'hm_home',EXP,'archive') -TMPOUT = os.path.join(SCRATCH,'tmp_festat') # Working directory 'tmp_festat' +TMPOUT = os.path.join(SCRATCH,'tmp_festat') # Working directory 'tmp_festat' os.makedirs(TMPOUT, mode=0o755, exist_ok=True) os.chdir(TMPOUT) -ens_mem=range(1,10) # Ensemble member (ens1,ens2.....,ens9) -filename = "ICMSHHARM+0006" # 6 hour forecast FA file +ens_mem=range(1,10) # Ensemble member (ens1,ens2.....,ens9) +filename = "ICMSHHARM+0006" # 6 hour forecast FA file fnumber=1 # -while dtgbeg <= dtgend: - YY=dtgbeg.isoformat()[0:4] - MM=dtgbeg.isoformat()[5:7] - DD=dtgbeg.isoformat()[8:10] - HH=dtgbeg.isoformat()[11:13] +while dtg <= dtgend: + # + YY = str(dtg.year).ljust(4, '0') + MM = str(dtg.month).zfill(2) + DD = str(dtg.day).zfill(2) + HH = str(dtg.hour).zfill(2) + # for mm in ens_mem: mem='mbr00'.ljust(6,str(mm)) target = os.path.join(HM_DATA, YY, MM, DD, HH, mem, filename) - print(target) - # - if ( fnumber <= 9 ): - filelink = filename[:-1]+str(fnumber) - elif ( fnumber >= 10 and fnumber <= 99 ): - filelink = filename[:-2]+str(fnumber) - elif ( fnumber >= 100 and fnumber <= 999 ): - filelink = filename[:-3]+str(fnumber) - else: - break - # - if os.path.isfile(filelink): - os.remove(filelink) - os.symlink(target ,filelink) + filelink = filename[:-4] + str(fnumber).zfill(4) + if os.path.isfile(str(filelink)): + os.remove(str(filelink)) + os.symlink(str(target),str(filelink)) + print(str(filelink),str(target)) fnumber += 1 - - dtgbeg += cycl + dtg += cycl # print(TMPOUT) quit() diff --git a/scripts/festat_S2.job b/scripts/festat_S2.job old mode 100644 new mode 100755 From 63e3efd837c6f3830ad976719de982ae25b2b381 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 09:44:41 +0000 Subject: [PATCH 05/20] Rename festat script and add to installation --- scripts/CMakeLists.txt | 1 + scripts/{festat_S2.job => runfestat.sh} | 0 2 files changed, 1 insertion(+) rename scripts/{festat_S2.job => runfestat.sh} (100%) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index cfab18d..8c00b93 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -11,5 +11,6 @@ endfunction() install_scripts (jbdiagnose.sh plotjbbal.py plotjbdiag.py) install_scripts (varbcdiagnose.sh plotvarbcpred.py) install_scripts (rundiacov.sh diag_diacov.sh plotdiacov.py) +install_scripts (runfestat.sh) install_scripts (dfscomp.sh plotdfs.py) #install_scripts (diacov.sh festat.sh) diff --git a/scripts/festat_S2.job b/scripts/runfestat.sh similarity index 100% rename from scripts/festat_S2.job rename to scripts/runfestat.sh From 8ec02820194f46f137b0ad569db9fff8d6e34364 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 10:44:26 +0000 Subject: [PATCH 06/20] Start generalising the run script --- scripts/runfestat.sh | 172 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 169 insertions(+), 3 deletions(-) diff --git a/scripts/runfestat.sh b/scripts/runfestat.sh index d5b24cf..0d825e9 100755 --- a/scripts/runfestat.sh +++ b/scripts/runfestat.sh @@ -1,4 +1,125 @@ #!/bin/bash + +bold=$(tput bold) +normal=$(tput sgr0) +unline=$(tput smul) + +usage() { + +PROGNAME=`basename $0` + +cat << EOFUSAGE + +${bold}NAME${normal} + ${PROGNAME} - Run FESTAT to ... + +${bold}USAGE${normal} + ${PROGNAME} -m -x -y + [ -t ] + [ -b ] + [ -i ] [ -o ] + [ -h ] + +${bold}DESCRIPTION${normal} + Script to run Forecast component of the AHNS. This script can create + batch submission headers and run a sample forecast. + + +${bold}OPTIONS${normal} + -m ${unline}host name${normal} + The name of your platform used in logic contained in this script. + [ECMWF|LOCAL] + -x ${unline}nprocx${normal} + Number of processors in x-direction for 2D domain decomposition + + -y ${unline}nprocy${normal} + Number of processors in y-direction for 2D domain decomposition + + -t ${unline}threads${normal} + Number of OpenMP threads [ default : 1 ] + + -b ${unline}binary-directory${normal} + PATH to FESTAT binary + + -i ${unline}input-directory${normal} + + -o ${unline}output-directory${normal} + +EOFUSAGE + +} + +NPROCX=-1 +NPROCY=-1 +THREADS=1 +HOST=ECMWF # Host ECMWF|LOCAL +BINDIR=$HOME/install/bin # Where to install libraries/executables +INPDIR=$HOME/festat_inputs # Where to install libraries/executables +OUTDIR=$HOME/festat_output # Where to install libraries/executables +UNIQTAG=$(mktemp -u XXXXXXXX) + +USAGE=0 + +while getopts m:x:y:t:b:i:o:h option +do + case $option in + m) + HOST=$OPTARG + ;; + x) + NPROCX=$OPTARG + ;; + y) + NPROCY=$OPTARG + ;; + t) + THREADS=$OPTARG + ;; + b) + BINDIR=$OPTARG + ;; + i) + INPDIR=$OPTARG + ;; + o) + OUTDIR=$OPTARG + ;; + h) + USAGE=1 + ;; + *) + USAGE=1 + ;; + esac +done + +if [ ${USAGE} -eq 1 -o "$#" -eq 0 ]; then + usage + exit 1 +fi + +if [ ${NPROCX} -lt 1 -o ${NPROCY} -lt 1 ]; then + echo "Invalid NPROC settings: ${NPROCX}, ${NPROCY}" + exit 1 +fi +# +# Calculate NPROC and NPROCXY +# +NPROCXY=`expr ${NPROCX} \* ${NPROCY}` +NPROC=${NPROCXY} + +WRK=${OUTDIR} +echo " ... output-directory: ${WRK}" + +WDIR=wrkFestat_${HOST}_${NPROCX}_${NPROCY}_${THREADS} +d_FESTAT=${WRK}/${WDIR} +echo " ... run-directory : ${d_FESTAT}" +mkdir -p ${d_FESTAT} +cd ${d_FESTAT} + +if [ ${HOST} == "ECMWF" ] ; then + + cat << EOFBATCH > batch.tmp #SBATCH --cpus-per-task=8 #SBATCH --error=festat.log #SBATCH --job-name=FESTAT @@ -8,7 +129,7 @@ #SBATCH --ntasks-per-node=27 #SBATCH --time=02:00:00 # -# + export MPPEXEC="srun" export NPROC=999 export NPROCX=1 @@ -20,10 +141,52 @@ eojcmd="" ulimit -S -s unlimited || ulimit -s ulimit -S -m unlimited || ulimit -m ulimit -S -d unlimited || ulimit -d +EOFBATCH + + SUBMIT=sbatch + +elif [ ${HOST} == "LOCAL" ] ; then + cat << EOFBATCH > batch.tmp +#-- host specific settings added by $0 +## +## INSERT batch directives here +## + +ulimit -S -s unlimited || ulimit -s +ulimit -S -m unlimited || ulimit -m +ulimit -S -d unlimited || ulimit -d + +#METIE +export DR_HOOK=1 +export DR_HOOK_IGNORE_SIGNALS=8 +export DR_HOOK_SILENT=1 # 0|1 +export DR_HOOK_OPT=prof # calls,cputime,walltime,times,heap,stack,rss + # paging,memory,all,prof,cpuprof,hpmprof,trim,self +export DR_HOOK_SHOW_PROCESS_OPTIONS=0 +export DR_HOOK_PROFILE=drhook.prof.%d + +export MPPEXEC="mpirun -np ${NPROC}" +#-- END host specific settings added by $0 + +EOFBATCH + + SUBMIT="" + +else + + echo "ERROR: host not supported ... exiting" + exit 1 + +fi + set -e ## EXP=SURF_CARRA2_CY46 ENV=${HOME}/hm_home/$EXP + +#source ${ENV}/Env_system + + # Build the namelist cat > fort.4 << EOF &NAMFESTAT @@ -42,8 +205,11 @@ cat > fort.4 << EOF OUTCVU='stab.cv', / EOF -source ${ENV}/Env_system -srun FESTAT || exit +echo "#!/bin/bash" > festat_batch.job +cat batch.tmp >> festat_batch.job +echo "${MPPEXEC} ${BINDIR}/FESTAT || exit " >> festat_batch.job + +${SUBMIT} ./festat_batch.job # trap - 0 exit From 802af7bb67eb56edbde207e599058c5c6f87d7de Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 10:56:45 +0000 Subject: [PATCH 07/20] Adjust creations and execution of the run script --- scripts/runfestat.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/runfestat.sh b/scripts/runfestat.sh index 0d825e9..a166b7e 100755 --- a/scripts/runfestat.sh +++ b/scripts/runfestat.sh @@ -205,11 +205,18 @@ cat > fort.4 << EOF OUTCVU='stab.cv', / EOF -echo "#!/bin/bash" > festat_batch.job -cat batch.tmp >> festat_batch.job -echo "${MPPEXEC} ${BINDIR}/FESTAT || exit " >> festat_batch.job -${SUBMIT} ./festat_batch.job +echo " ... add system specific settings and required environment variables to a run script" +echo "#!/bin/bash" > festat_${PARTITION}_${NPROCX}_${NPROCY}_${THREADS}.sh +cat batch.tmp >> festat_${PARTITION}_${NPROCX}_${NPROCY}_${THREADS}.sh +echo "${MPPEXEC} ${BINDIR}/FESTAT || exit " >> festat_${PARTITION}_${NPROCX}_${NPROCY}_${THREADS}.sh +chmod 755 festat_${PARTITION}_${NPROCX}_${NPROCY}_${THREADS}.sh + +echo " ... submit job with ... " +echo " ... ${SUBMIT} ./festat_${PARTITION}_${NPROCX}_${NPROCY}_${THREADS}.sh" +echo " " + +${SUBMIT} ./festat_${PARTITION}_${NPROCX}_${NPROCY}_${THREADS}.sh # trap - 0 exit From ef6287b196c65dc88db82455f951b8f621e80a8d Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 10:59:09 +0000 Subject: [PATCH 08/20] Move env setting in to ECMWF batch settings --- scripts/runfestat.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/runfestat.sh b/scripts/runfestat.sh index a166b7e..75ccc20 100755 --- a/scripts/runfestat.sh +++ b/scripts/runfestat.sh @@ -49,6 +49,8 @@ EOFUSAGE } +set -e + NPROCX=-1 NPROCY=-1 THREADS=1 @@ -130,6 +132,10 @@ if [ ${HOST} == "ECMWF" ] ; then #SBATCH --time=02:00:00 # +EXP=SURF_CARRA2_CY46 +ENV=${HOME}/hm_home/$EXP +source ${ENV}/Env_system + export MPPEXEC="srun" export NPROC=999 export NPROCX=1 @@ -179,14 +185,6 @@ else fi -set -e -## -EXP=SURF_CARRA2_CY46 -ENV=${HOME}/hm_home/$EXP - -#source ${ENV}/Env_system - - # Build the namelist cat > fort.4 << EOF &NAMFESTAT From 5f834fd565451c91da66a69f36f48ba41766692d Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 11:09:57 +0000 Subject: [PATCH 09/20] Start adding Swapan's documentation --- doc/tools/festat.md | 83 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 doc/tools/festat.md diff --git a/doc/tools/festat.md b/doc/tools/festat.md new file mode 100644 index 0000000..84bd044 --- /dev/null +++ b/doc/tools/festat.md @@ -0,0 +1,83 @@ +# FESTAT- Forecast Error statistics: A Technical Summary + +## Introduction: + +The background error statistics (BGE), generally referred to as structure functions, are produced through the standalone FESTAT (Forecast Error Statistics) method. The FESTAT for FA test has been completed and is currently accessible in cycle CY46h1. To mitigate the substantial storage problem, the B-matrix is computed in terms of control vectors, assuming statistical independence among them. The control variables encompass vorticity, divergence, specific humidity, surface pressure, and temperature. + + VDF (Forecast Error Vector) ----------> Control Vector + +Here, F represents the horizontal 2-dimensional Fourier transform, transitioning from physical grid point space to spectral space. D functions as a balancing operator, and V is a vertical transform that utilizes the eigen-vectors of vertical covariance matrices. + +The spectral transformation (denoted as F) is essential because certain fields are expressed in grid points rather than spectral coefficients. Moreover, FESTAT requires vorticity and divergence derived from geographical wind components in the input FA file. To calculate vorticity and divergence, the initial step involves computing, bi-periodicizing, and smoothing the map factor in spectral space. Subsequently, the wind is reduced by the map factor in gridpoint space. Finally, the wind components are transformed into vorticity and divergence. + +The statistical de-correlation operator D, also known as the balance operator, is derived in spectral space using multiple regression techniques, performed separately for each wavenumber component. The multiple regression operates stepwise, commencing with the background forecast errors of vorticity, divergence, temperature, surface pressure, and specific humidity in spectral space. In summary, the FESTAT software manages multiple tasks related to the error statistics, including (1) converting variables from gridpoint space to spectral point space, (2) computing the balance operator, (3) generating horizontal variance density spectra for control variables, and (4) determining vertical correlations for the control variables. + + +## Fortran code structure for FESTAT: +Figure 1. Flowchart of FASTEM Fortran code. +```mermaid +graph LR + +``` +here, +- A. Festet.F90 is the main Fortran probeam that has two subroutine (B, C) +- B. Subroutine SUFESTAT (sufestat.F90) handle namelist (fort.4) and MPI setup for the program FESTAT. This Fortran program also check the file type FA or GRIB2 +- C. Subroutine NMCSTAT (nmcstat.F90) mainly handles error statistics, covariances using NMC forecast differences. This subroutine is called multiple subroutines to calculate forecast difference and the error statistics. Few of them are listed below. +- C1. Subroutine SUFESPEC (sufespec.F90) reads input file (CNAME = 'ICMSHHARM+0') either in FA or GRIB2 format. +- C2. Subroutine EREGPDIV (eregpdiv.F90) is used to estimate empirical vertical balance matrix pmat by a linear regression between balanced mass Pb and div. The input data is the horizontal balance coefficient (fact1). The output vertical coefficient matrix for creating balanced Div (PMAT). In addition to that the program writes the total cross-covariance matrix ('covdp.xy'). +- C3. Subroutine EREGPDT (eregpdt.F90) Estimate empirical vertical balance matrix stps by a linear regression between (balanced mass, unbalanced div) and (T,ps). The input sets of data are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix for creating balanced Div (sdiv). The output data are the vertical coefficient matrix for creating balanced T, lnPs (stps). This subroutine generates output files naming ‘covtp.xy’, ‘cortp.xy’, ‘covpsp.y’, ‘covtd.xy’ and ‘covpsd.y’ . +- C4. Subroutine EREGPDTQ (eregpdtq.F90) estimates empirical vertical balance matrix pmat by a linear regression between (Pb, divu, Tpsu) and q. The input data sets are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix to define balanced div and T, lnP (sdiv and stps). The output data are the vertical coefficient matrix for creating balanced q. This subroutine generates output files naming 'covqp.xy', 'covqd.xy', 'covqt.xy'. +- C5. Subroutine EWGSABAL (ewgsabal.F90) writes a GSA file with the balance linear regression coefficient for stabal96 Jb formulation. +- C6. Subroutine ECALCOV (ecalcov.F90) Compute covariances as defined by the balance and write them on stat files. The input datasets are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix to define balanced q (sq), and (c) vertical coefficient matrix to define balanced div and T,Ps. This subroutine generates output files naming 'expltlev.y', 'expldlev.y', 'explqlev.y', 'explq_pb.xy', explq_divu.xy', 'expl, q_tpsu.xy', 'explq_wave.y', 'explq_lev.y', 'expltps_pb.xy', 'expltps_divu.xy', 'explt_wave.y', 'explps_wave.y', 'explt_lev.y'. + +## FESTAT stand-alone script for FA: + +The festat standalone script runfestat reading FA-files has been tested with nine ensemble members. An example of how to run the Festat standalone script with a total 999 files is described below. + +A total 999 historical FA files were initialized at the assimilation cycle (00, 06 12 and 18 UTC) and for each day. For each assimilation cycle, 9 ensemble forecasts are generated separately. The Variational Data Assimilation (3D-VAR) method is used for the observation assimilation of each ensemble member. + +Hence, total FA file is equal to 9 (ens member) X 111 (assimilation cycle)= 999 FA files (6-hr forecast). + +=> NCASES = 999 + +The _runfestat_ script for FA is driven by a namelist file (fort.4) in the current working directory. + +``` +&NAMFESTAT + NCASES=999, + LSTABAL=.true., + LANAFBAL=.true., + LELAM=.TRUE., + CNAME='ICMSHHARM+0', + NFRGRP=9, + NPRINTLEV=0, + LOZONE=.FALSE., + LOZBAL=.FALSE., + LUVBAL=.FALSE., + OUTBAL='stab.bal' + OUTCVT='stab.cvt', + OUTCVU='stab.cv', +/ +``` + +where +NCASES = Number of cases (total number of file) +NFRGRP = Number of members per group of files. + + + +The nine ensemble member (FA files) are organized into 111 groups. Each member "n" within a group is computed as the difference between the historical files "n+m" and "n," except for the last one. The final member of a group is calculated as the difference between FA files "n" and "m" + +It's important to note that the current FORTRAN code is designed for a maximum of 999 FA files. If the number exceeds 999, the sufespeca.F90 program will terminate with the following message. + + +## References: + +1. https://hirlam.github.io/HarmonieSystemDocumentation/dev/DataAssimilation/StructureFunctions/ +2. https://opensource.umr-cnrm.fr/attachments/5000/festat_guidelines.pdf +3. https://opensource.umr-cnrm.fr/attachments/4279/festat_for_fa.pdf +4. https://www.tandfonline.com/doi/full/10.1080/16000870.2019.1615168 + + + + From 1986509550a43c6b58838297bab45a376bd90184 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 11:16:23 +0000 Subject: [PATCH 10/20] Start code diagram --- doc/tools/festat.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index 84bd044..7c94e18 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -16,7 +16,11 @@ The statistical de-correlation operator D, also known as the balance operator, i ## Fortran code structure for FESTAT: Figure 1. Flowchart of FASTEM Fortran code. ```mermaid -graph LR +classDiagram + +FESTAT <|-- SUFESTAT +FESTAT <|-- NMCSTAT + ``` here, From 25b6c6f57bd06c8f8e42dbc93079130a69d33ca7 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 11:20:03 +0000 Subject: [PATCH 11/20] Update code diagram --- doc/tools/festat.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index 7c94e18..4ad4626 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -18,10 +18,16 @@ Figure 1. Flowchart of FASTEM Fortran code. ```mermaid classDiagram -FESTAT <|-- SUFESTAT -FESTAT <|-- NMCSTAT - - +FESTAT --|> SUFESTAT +FESTAT --|> NMCSTAT +NMCSTAT --|> SUFESPEC +NMCSTAT --|> EREGPDIV +NMCSTAT --|> EREGPDT +NMCSTAT --|> EREGPDTQ +NMCSTAT --|> EWGSABAL +NMCSTAT --|> ECALCOV +SUFESPEC --|> SUFESPECA +SUFESPEC --|> SUFESPECG1 ``` here, - A. Festet.F90 is the main Fortran probeam that has two subroutine (B, C) From c3eff36ca8ea5d0eb580f63b786c68a1c94de91d Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 11:21:11 +0000 Subject: [PATCH 12/20] Add note code diagram --- doc/tools/festat.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index 4ad4626..f47af8e 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -18,6 +18,8 @@ Figure 1. Flowchart of FASTEM Fortran code. ```mermaid classDiagram +note "FESTAT FORTRAN code structure" + FESTAT --|> SUFESTAT FESTAT --|> NMCSTAT NMCSTAT --|> SUFESPEC From 6c37e5d42cbb3d769282b43917c8f4bad2e8d243 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 11:27:47 +0000 Subject: [PATCH 13/20] Add code classification --- doc/tools/festat.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index f47af8e..a584359 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -14,7 +14,7 @@ The statistical de-correlation operator D, also known as the balance operator, i ## Fortran code structure for FESTAT: -Figure 1. Flowchart of FASTEM Fortran code. + ```mermaid classDiagram @@ -30,9 +30,21 @@ NMCSTAT --|> EWGSABAL NMCSTAT --|> ECALCOV SUFESPEC --|> SUFESPECA SUFESPEC --|> SUFESPECG1 + +NMCSTAT: PROGRAM() +SUFESTAT: SUBROUTINE() +NMCSTAT: SUBROUTINE() +SUFESPEC: SUBROUTINE() +EREGPDIV: SUBROUTINE() +EREGPDT: SUBROUTINE() +EREGPDTQ: SUBROUTINE() +EWGSABAL: SUBROUTINE() +ECALCOV: SUBROUTINE() +SUFESPECA: SUBROUTINE() +SUFESPECG1: SUBROUTINE() ``` here, -- A. Festet.F90 is the main Fortran probeam that has two subroutine (B, C) +- A. Festat.F90 is the main Fortran program that has two subroutine (B, C) - B. Subroutine SUFESTAT (sufestat.F90) handle namelist (fort.4) and MPI setup for the program FESTAT. This Fortran program also check the file type FA or GRIB2 - C. Subroutine NMCSTAT (nmcstat.F90) mainly handles error statistics, covariances using NMC forecast differences. This subroutine is called multiple subroutines to calculate forecast difference and the error statistics. Few of them are listed below. - C1. Subroutine SUFESPEC (sufespec.F90) reads input file (CNAME = 'ICMSHHARM+0') either in FA or GRIB2 format. From a066db07dcc1c4ead86f3cbc6e4f129399ca0210 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 11:28:22 +0000 Subject: [PATCH 14/20] Correct code classification --- doc/tools/festat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index a584359..1fa4a82 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -31,7 +31,7 @@ NMCSTAT --|> ECALCOV SUFESPEC --|> SUFESPECA SUFESPEC --|> SUFESPECG1 -NMCSTAT: PROGRAM() +FESTAT: PROGRAM() SUFESTAT: SUBROUTINE() NMCSTAT: SUBROUTINE() SUFESPEC: SUBROUTINE() From 9e6f8bc0d824b83ff0fb62d8c91bd028847fc4b1 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 12:08:36 +0000 Subject: [PATCH 15/20] Add help to run script doc --- doc/tools/festat.md | 119 ++++++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 38 deletions(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index 1fa4a82..85ef950 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -12,51 +12,52 @@ The spectral transformation (denoted as F) is essential because certain fields a The statistical de-correlation operator D, also known as the balance operator, is derived in spectral space using multiple regression techniques, performed separately for each wavenumber component. The multiple regression operates stepwise, commencing with the background forecast errors of vorticity, divergence, temperature, surface pressure, and specific humidity in spectral space. In summary, the FESTAT software manages multiple tasks related to the error statistics, including (1) converting variables from gridpoint space to spectral point space, (2) computing the balance operator, (3) generating horizontal variance density spectra for control variables, and (4) determining vertical correlations for the control variables. +## The FESTAT run script -## Fortran code structure for FESTAT: +The festat standalone script _runfestat_ reading FA-files has been tested with nine ensemble members. An example of how to run the Festat standalone script with a total 999 files is described below. +### Options +``` +[user@localhost ~]$ runfestat -h -```mermaid -classDiagram +NAME + runfestat.sh - Run FESTAT to ... -note "FESTAT FORTRAN code structure" +USAGE + runfestat.sh -m -x -y + [ -t ] + [ -b ] + [ -i ] [ -o ] + [ -h ] -FESTAT --|> SUFESTAT -FESTAT --|> NMCSTAT -NMCSTAT --|> SUFESPEC -NMCSTAT --|> EREGPDIV -NMCSTAT --|> EREGPDT -NMCSTAT --|> EREGPDTQ -NMCSTAT --|> EWGSABAL -NMCSTAT --|> ECALCOV -SUFESPEC --|> SUFESPECA -SUFESPEC --|> SUFESPECG1 +DESCRIPTION + Script to run Forecast component of the AHNS. This script can create + batch submission headers and run a sample forecast. -FESTAT: PROGRAM() -SUFESTAT: SUBROUTINE() -NMCSTAT: SUBROUTINE() -SUFESPEC: SUBROUTINE() -EREGPDIV: SUBROUTINE() -EREGPDT: SUBROUTINE() -EREGPDTQ: SUBROUTINE() -EWGSABAL: SUBROUTINE() -ECALCOV: SUBROUTINE() -SUFESPECA: SUBROUTINE() -SUFESPECG1: SUBROUTINE() -``` -here, -- A. Festat.F90 is the main Fortran program that has two subroutine (B, C) -- B. Subroutine SUFESTAT (sufestat.F90) handle namelist (fort.4) and MPI setup for the program FESTAT. This Fortran program also check the file type FA or GRIB2 -- C. Subroutine NMCSTAT (nmcstat.F90) mainly handles error statistics, covariances using NMC forecast differences. This subroutine is called multiple subroutines to calculate forecast difference and the error statistics. Few of them are listed below. -- C1. Subroutine SUFESPEC (sufespec.F90) reads input file (CNAME = 'ICMSHHARM+0') either in FA or GRIB2 format. -- C2. Subroutine EREGPDIV (eregpdiv.F90) is used to estimate empirical vertical balance matrix pmat by a linear regression between balanced mass Pb and div. The input data is the horizontal balance coefficient (fact1). The output vertical coefficient matrix for creating balanced Div (PMAT). In addition to that the program writes the total cross-covariance matrix ('covdp.xy'). -- C3. Subroutine EREGPDT (eregpdt.F90) Estimate empirical vertical balance matrix stps by a linear regression between (balanced mass, unbalanced div) and (T,ps). The input sets of data are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix for creating balanced Div (sdiv). The output data are the vertical coefficient matrix for creating balanced T, lnPs (stps). This subroutine generates output files naming ‘covtp.xy’, ‘cortp.xy’, ‘covpsp.y’, ‘covtd.xy’ and ‘covpsd.y’ . -- C4. Subroutine EREGPDTQ (eregpdtq.F90) estimates empirical vertical balance matrix pmat by a linear regression between (Pb, divu, Tpsu) and q. The input data sets are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix to define balanced div and T, lnP (sdiv and stps). The output data are the vertical coefficient matrix for creating balanced q. This subroutine generates output files naming 'covqp.xy', 'covqd.xy', 'covqt.xy'. -- C5. Subroutine EWGSABAL (ewgsabal.F90) writes a GSA file with the balance linear regression coefficient for stabal96 Jb formulation. -- C6. Subroutine ECALCOV (ecalcov.F90) Compute covariances as defined by the balance and write them on stat files. The input datasets are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix to define balanced q (sq), and (c) vertical coefficient matrix to define balanced div and T,Ps. This subroutine generates output files naming 'expltlev.y', 'expldlev.y', 'explqlev.y', 'explq_pb.xy', explq_divu.xy', 'expl, q_tpsu.xy', 'explq_wave.y', 'explq_lev.y', 'expltps_pb.xy', 'expltps_divu.xy', 'explt_wave.y', 'explps_wave.y', 'explt_lev.y'. -## FESTAT stand-alone script for FA: +OPTIONS + -m host name + The name of your platform used in logic contained in this script. + [ECMWF|LOCAL] + -x nprocx + Number of processors in x-direction for 2D domain decomposition + + -y nprocy + Number of processors in y-direction for 2D domain decomposition + + -t threads + Number of OpenMP threads [ default : 1 ] + + -b binary-directory + PATH to FESTAT binary + + -i input-directory + + -o output-directory + +[user@localhost ~]$ +``` -The festat standalone script runfestat reading FA-files has been tested with nine ensemble members. An example of how to run the Festat standalone script with a total 999 files is described below. +### Example A total 999 historical FA files were initialized at the assimilation cycle (00, 06 12 and 18 UTC) and for each day. For each assimilation cycle, 9 ensemble forecasts are generated separately. The Variational Data Assimilation (3D-VAR) method is used for the observation assimilation of each ensemble member. @@ -95,6 +96,48 @@ The nine ensemble member (FA files) are organized into 111 groups. Each member " It's important to note that the current FORTRAN code is designed for a maximum of 999 FA files. If the number exceeds 999, the sufespeca.F90 program will terminate with the following message. +## Fortran code structure for FESTAT: + +```mermaid +classDiagram + +note "FESTAT FORTRAN code structure" + +FESTAT --|> SUFESTAT +FESTAT --|> NMCSTAT +NMCSTAT --|> SUFESPEC +NMCSTAT --|> EREGPDIV +NMCSTAT --|> EREGPDT +NMCSTAT --|> EREGPDTQ +NMCSTAT --|> EWGSABAL +NMCSTAT --|> ECALCOV +SUFESPEC --|> SUFESPECA +SUFESPEC --|> SUFESPECG1 + +FESTAT: PROGRAM() +SUFESTAT: SUBROUTINE() +NMCSTAT: SUBROUTINE() +SUFESPEC: SUBROUTINE() +EREGPDIV: SUBROUTINE() +EREGPDT: SUBROUTINE() +EREGPDTQ: SUBROUTINE() +EWGSABAL: SUBROUTINE() +ECALCOV: SUBROUTINE() +SUFESPECA: SUBROUTINE() +SUFESPECG1: SUBROUTINE() +``` +here, +- A. FESTAT is the main FORTRAN program that calls two subroutine, SUFESTAT and NMCSTAT +- B. Subroutine SUFESTAT (sufestat.F90) handles the namelist (fort.4) and MPI setup for the program FESTAT. This subroutine program also checks whic file type (FA or GRIB2) to read. +- C. Subroutine NMCSTAT (nmcstat.F90) mainly handles error statistics, covariances using NMC forecast differences. This subroutine is calls multiple subroutines to calculate forecast difference and the error statistics. Some of them are listed below. +- C1. Subroutine SUFESPEC (sufespec.F90) reads input files (CNAME = 'ICMSHHARM+0') either in FA or GRIB2 format. +- C2. Subroutine EREGPDIV (eregpdiv.F90) is used to estimate empirical vertical balance matrix pmat by a linear regression between balanced mass Pb and div. The input data is the horizontal balance coefficient (fact1). The output vertical coefficient matrix for creating balanced Div (PMAT). In addition to that the program writes the total cross-covariance matrix ('covdp.xy'). +- C3. Subroutine EREGPDT (eregpdt.F90) Estimate empirical vertical balance matrix stps by a linear regression between (balanced mass, unbalanced div) and (T,ps). The input sets of data are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix for creating balanced Div (sdiv). The output data are the vertical coefficient matrix for creating balanced T, lnPs (stps). This subroutine generates output files naming ‘covtp.xy’, ‘cortp.xy’, ‘covpsp.y’, ‘covtd.xy’ and ‘covpsd.y’ . +- C4. Subroutine EREGPDTQ (eregpdtq.F90) estimates empirical vertical balance matrix pmat by a linear regression between (Pb, divu, Tpsu) and q. The input data sets are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix to define balanced div and T, lnP (sdiv and stps). The output data are the vertical coefficient matrix for creating balanced q. This subroutine generates output files naming 'covqp.xy', 'covqd.xy', 'covqt.xy'. +- C5. Subroutine EWGSABAL (ewgsabal.F90) writes a GSA file with the balance linear regression coefficient for stabal96 Jb formulation. +- C6. Subroutine ECALCOV (ecalcov.F90) Compute covariances as defined by the balance and write them on stat files. The input datasets are (a) horizontal balance coefficient (fact1) and (b) vertical coefficient matrix to define balanced q (sq), and (c) vertical coefficient matrix to define balanced div and T,Ps. This subroutine generates output files naming 'expltlev.y', 'expldlev.y', 'explqlev.y', 'explq_pb.xy', explq_divu.xy', 'expl, q_tpsu.xy', 'explq_wave.y', 'explq_lev.y', 'expltps_pb.xy', 'expltps_divu.xy', 'explt_wave.y', 'explps_wave.y', 'explt_lev.y'. + + ## References: 1. https://hirlam.github.io/HarmonieSystemDocumentation/dev/DataAssimilation/StructureFunctions/ From b6a57f82f80f3f69a988fa9c30f551b4889c8caf Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 13:07:23 +0000 Subject: [PATCH 16/20] Put namelist variables in table --- doc/tools/festat.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index 85ef950..e37cba5 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -1,16 +1,16 @@ -# FESTAT- Forecast Error statistics: A Technical Summary +# FESTAT - Forecast Error STATistics: A Technical Summary -## Introduction: +## Introduction -The background error statistics (BGE), generally referred to as structure functions, are produced through the standalone FESTAT (Forecast Error Statistics) method. The FESTAT for FA test has been completed and is currently accessible in cycle CY46h1. To mitigate the substantial storage problem, the B-matrix is computed in terms of control vectors, assuming statistical independence among them. The control variables encompass vorticity, divergence, specific humidity, surface pressure, and temperature. +The background error statistics (BGE), generally referred to as structure functions, are produced using the standalone FESTAT (Forecast Error Statistics) method. The FESTAT for FA test has been completed and is currently accessible in cycle CY46h1. To mitigate the substantial storage problem, the B-matrix is computed in terms of control vectors, assuming statistical independence among them. The control variables encompass vorticity, divergence, specific humidity, surface pressure, and temperature. - VDF (Forecast Error Vector) ----------> Control Vector + **V****D**F** (Forecast Error Vector) ----------> Control Vector -Here, F represents the horizontal 2-dimensional Fourier transform, transitioning from physical grid point space to spectral space. D functions as a balancing operator, and V is a vertical transform that utilizes the eigen-vectors of vertical covariance matrices. +Here, **F** represents the horizontal 2-dimensional Fourier transform, transitioning from physical grid point space to spectral space. **D** functions as a balancing operator, and **V** is a vertical transform that utilizes the eigen-vectors of vertical covariance matrices. -The spectral transformation (denoted as F) is essential because certain fields are expressed in grid points rather than spectral coefficients. Moreover, FESTAT requires vorticity and divergence derived from geographical wind components in the input FA file. To calculate vorticity and divergence, the initial step involves computing, bi-periodicizing, and smoothing the map factor in spectral space. Subsequently, the wind is reduced by the map factor in gridpoint space. Finally, the wind components are transformed into vorticity and divergence. +The spectral transformation (denoted as **F**) is essential because certain fields are expressed in grid points rather than spectral coefficients. Moreover, FESTAT requires vorticity and divergence derived from geographical wind components in the input FA file. To calculate vorticity and divergence, the initial step involves computing, bi-periodicizing, and smoothing the map factor in spectral space. Subsequently, the wind is reduced by the map factor in gridpoint space. Finally, the wind components are transformed into vorticity and divergence. -The statistical de-correlation operator D, also known as the balance operator, is derived in spectral space using multiple regression techniques, performed separately for each wavenumber component. The multiple regression operates stepwise, commencing with the background forecast errors of vorticity, divergence, temperature, surface pressure, and specific humidity in spectral space. In summary, the FESTAT software manages multiple tasks related to the error statistics, including (1) converting variables from gridpoint space to spectral point space, (2) computing the balance operator, (3) generating horizontal variance density spectra for control variables, and (4) determining vertical correlations for the control variables. +The statistical de-correlation operator **D**, also known as the balance operator, is derived in spectral space using multiple regression techniques, performed separately for each wavenumber component. The multiple regression operates stepwise, commencing with the background forecast errors of vorticity, divergence, temperature, surface pressure, and specific humidity in spectral space. In summary, the FESTAT software manages multiple tasks related to the error statistics, including (1) converting variables from gridpoint space to spectral point space, (2) computing the balance operator, (3) generating horizontal variance density spectra for control variables, and (4) determining vertical correlations for the control variables. ## The FESTAT run script @@ -86,8 +86,12 @@ The _runfestat_ script for FA is driven by a namelist file (fort.4) in the curre ``` where -NCASES = Number of cases (total number of file) -NFRGRP = Number of members per group of files. + +|= Name =|= Description =| +|--------|---------------| +|NCASES |CNAME which is set to fit the name of the native historical files. The actual names of the files will be CNAME completed with a number on 3 digits (from 1 to NCASES). | +|NFRGRP |NFRGRP represents the number of members per group of files. This variable is serving the former need of femars. The ensemble of files is divided into groups. Each member n of a group, except the last one, is computed as the difference between the historical files n+1 and n ; the latest member of a group is computed as the difference between the historical files 1 and n. In that example the number of members in each group is NFRGRP=6. Consequently the number of groups here is 9. | +| From 59f6fe272d3114e78af90a51d3938d0e14892736 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 13:09:29 +0000 Subject: [PATCH 17/20] Adjust tables definition --- doc/tools/festat.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index e37cba5..1a7e774 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -87,8 +87,8 @@ The _runfestat_ script for FA is driven by a namelist file (fort.4) in the curre where -|= Name =|= Description =| -|--------|---------------| +| Name | Description | +|:-------|:--------------| |NCASES |CNAME which is set to fit the name of the native historical files. The actual names of the files will be CNAME completed with a number on 3 digits (from 1 to NCASES). | |NFRGRP |NFRGRP represents the number of members per group of files. This variable is serving the former need of femars. The ensemble of files is divided into groups. Each member n of a group, except the last one, is computed as the difference between the historical files n+1 and n ; the latest member of a group is computed as the difference between the historical files 1 and n. In that example the number of members in each group is NFRGRP=6. Consequently the number of groups here is 9. | | From 91a65db3c79eac3c8e41384206ae318c8f3e472b Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 13:41:06 +0000 Subject: [PATCH 18/20] More namelist infor for doc --- doc/tools/festat.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index 1a7e774..7f342b0 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -4,7 +4,7 @@ The background error statistics (BGE), generally referred to as structure functions, are produced using the standalone FESTAT (Forecast Error Statistics) method. The FESTAT for FA test has been completed and is currently accessible in cycle CY46h1. To mitigate the substantial storage problem, the B-matrix is computed in terms of control vectors, assuming statistical independence among them. The control variables encompass vorticity, divergence, specific humidity, surface pressure, and temperature. - **V****D**F** (Forecast Error Vector) ----------> Control Vector + **V** **D** **F** (Forecast Error Vector) ----------> Control Vector Here, **F** represents the horizontal 2-dimensional Fourier transform, transitioning from physical grid point space to spectral space. **D** functions as a balancing operator, and **V** is a vertical transform that utilizes the eigen-vectors of vertical covariance matrices. @@ -87,12 +87,22 @@ The _runfestat_ script for FA is driven by a namelist file (fort.4) in the curre where -| Name | Description | -|:-------|:--------------| -|NCASES |CNAME which is set to fit the name of the native historical files. The actual names of the files will be CNAME completed with a number on 3 digits (from 1 to NCASES). | -|NFRGRP |NFRGRP represents the number of members per group of files. This variable is serving the former need of femars. The ensemble of files is divided into groups. Each member n of a group, except the last one, is computed as the difference between the historical files n+1 and n ; the latest member of a group is computed as the difference between the historical files 1 and n. In that example the number of members in each group is NFRGRP=6. Consequently the number of groups here is 9. | -| - +| Name | Description | +|:-----------|:--------------| +|NCASES |Number of cases; i.e. the total number of files | +|NFRGRP |NFRGRP represents the number of members per group of files. This variable is serving the former need of femars. The ensemble of files is divided into groups. Each member n of a group, except the last one, is computed as the difference between the historical files n+1 and n ; the latest member of a group is computed as the difference between the historical files 1 and n. In that example the number of members in each group is NFRGRP=6. Consequently the number of groups here is 9. | +|CNAME |CNAME which is set to fit the name of the native historical files. The actual names of the files will be CNAME completed with a number on 3 digits (from 1 to NCASES). | +|LSTABAL |logical switch to compute balance statistics | +|LANAFBAL |logical switch to compute analytic f-plane horizontal balance for vorticity | +|LELAM |logical switch to indicate LAM model (or not) | +|LOZONE |logical switch to process O3 covariances | +|LOZBAL |logical switch to put O3 covariances in the balance | +|LUVBAL |logical switch to activate the reading of mean wind (LAM model) | +|LNETCDF_IO |logical switch to output in netcdf for bal/cv files | +|NPRINTLEV |verbosity setting (integer) | +|OUTBAL |filename for balance linear regression coefficients | +|OUTCVT |filename for vertical error covariances? | +|OUTCVU |filename for vertical error covariances? | The nine ensemble member (FA files) are organized into 111 groups. Each member "n" within a group is computed as the difference between the historical files "n+m" and "n," except for the last one. The final member of a group is calculated as the difference between FA files "n" and "m" From 4131736a2d7b804668d2a93e17e796164f6207a5 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 13:53:15 +0000 Subject: [PATCH 19/20] Improved FESTAT math --- doc/tools/festat.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/tools/festat.md b/doc/tools/festat.md index 7f342b0..0c2d4ec 100644 --- a/doc/tools/festat.md +++ b/doc/tools/festat.md @@ -1,10 +1,11 @@ -# FESTAT - Forecast Error STATistics: A Technical Summary +# FESTAT - Forecast Error STATistics ## Introduction -The background error statistics (BGE), generally referred to as structure functions, are produced using the standalone FESTAT (Forecast Error Statistics) method. The FESTAT for FA test has been completed and is currently accessible in cycle CY46h1. To mitigate the substantial storage problem, the B-matrix is computed in terms of control vectors, assuming statistical independence among them. The control variables encompass vorticity, divergence, specific humidity, surface pressure, and temperature. - - **V** **D** **F** (Forecast Error Vector) ----------> Control Vector +The background error (BGE) statistics, generally referred to as structure functions, are produced using the standalone FESTAT (Forecast Error Statistics) method. The FESTAT for FA test has been completed and is currently accessible in cycle CY46h1. To mitigate the substantial storage problem, the B-matrix is computed in terms of control vectors, assuming statistical independence among them. The control variables encompass vorticity, divergence, specific humidity, surface pressure, and temperature. The forecast BGE vector, delta chi, is transformed to a control vector X: +```math + \chi = VDF \delta X +``` Here, **F** represents the horizontal 2-dimensional Fourier transform, transitioning from physical grid point space to spectral space. **D** functions as a balancing operator, and **V** is a vertical transform that utilizes the eigen-vectors of vertical covariance matrices. @@ -12,7 +13,7 @@ The spectral transformation (denoted as **F**) is essential because certain fiel The statistical de-correlation operator **D**, also known as the balance operator, is derived in spectral space using multiple regression techniques, performed separately for each wavenumber component. The multiple regression operates stepwise, commencing with the background forecast errors of vorticity, divergence, temperature, surface pressure, and specific humidity in spectral space. In summary, the FESTAT software manages multiple tasks related to the error statistics, including (1) converting variables from gridpoint space to spectral point space, (2) computing the balance operator, (3) generating horizontal variance density spectra for control variables, and (4) determining vertical correlations for the control variables. -## The FESTAT run script +## The run script The festat standalone script _runfestat_ reading FA-files has been tested with nine ensemble members. An example of how to run the Festat standalone script with a total 999 files is described below. ### Options @@ -110,7 +111,7 @@ The nine ensemble member (FA files) are organized into 111 groups. Each member " It's important to note that the current FORTRAN code is designed for a maximum of 999 FA files. If the number exceeds 999, the sufespeca.F90 program will terminate with the following message. -## Fortran code structure for FESTAT: +## Fortran code ```mermaid classDiagram From ef5bf4473d903f3c9f07486622899a7ff2b61cc0 Mon Sep 17 00:00:00 2001 From: Eoin Whelan Date: Tue, 13 Feb 2024 14:16:53 +0000 Subject: [PATCH 20/20] Cosmetic script changes --- scripts/runfestat.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/runfestat.sh b/scripts/runfestat.sh index 75ccc20..a09f31a 100755 --- a/scripts/runfestat.sh +++ b/scripts/runfestat.sh @@ -31,10 +31,10 @@ ${bold}OPTIONS${normal} [ECMWF|LOCAL] -x ${unline}nprocx${normal} Number of processors in x-direction for 2D domain decomposition - + -y ${unline}nprocy${normal} Number of processors in y-direction for 2D domain decomposition - + -t ${unline}threads${normal} Number of OpenMP threads [ default : 1 ] @@ -188,13 +188,13 @@ fi # Build the namelist cat > fort.4 << EOF &NAMFESTAT + NPRINTLEV=0, NCASES=$NPROC, - LSTABAL=.true., - LANAFBAL=.true., - LELAM=.TRUE., - CNAME='ICMSHHARM+0', NFRGRP=9, - NPRINTLEV=0, + CNAME='ICMSHHARM+0', + LSTABAL=.TRUE., + LANAFBAL=.TRUE., + LELAM=.TRUE., LOZONE=.FALSE., LOZBAL=.FALSE., LUVBAL=.FALSE.,