From a5f85a9832535a7e7eca19dfeab6e11446a6d836 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 18:52:44 +0000 Subject: [PATCH 1/9] archive and scrub lbcs --- parm/wflow/archiving.yaml | 22 ++++++++++++++++++++++ parm/wflow/scrubbing.yaml | 17 ++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/parm/wflow/archiving.yaml b/parm/wflow/archiving.yaml index eb4f6c7..ee58264 100644 --- a/parm/wflow/archiving.yaml +++ b/parm/wflow/archiving.yaml @@ -68,3 +68,25 @@ workflow: metataskdep: attrs: metatask: post + + task_archive_lbcs: + command: + cyclestr: + value: 'source &MPAS_APP;/load_wflow_modules.sh &PLATFORM; && + cd {{ user.experiment_dir }} && + &MPAS_APP;/scripts/archive_lbcs.sh @Y@m@d@H "&ARCHIVE_DIR;"' + account: "{{ platform.account }}" + partition: "{{ platform.service_partition }}" + join: + cyclestr: + value: '&LOGDIR;/{{ jobname }}_@Y@m@d@H.log' + walltime: "{{ archiving.lbcs.execution.walltime }}" + cores: !int "{{ archiving.lbcs.execution.cores }}" + dependency: + or: + taskdep: + attrs: + task: mpas + metataskdep: + attrs: + metatask: post diff --git a/parm/wflow/scrubbing.yaml b/parm/wflow/scrubbing.yaml index 249f571..7d5d2bf 100644 --- a/parm/wflow/scrubbing.yaml +++ b/parm/wflow/scrubbing.yaml @@ -22,7 +22,22 @@ workflow: maxtries: 2 command: cyclestr: - value: 'set -xue ; rm -rf "{{ user.experiment_dir }}/@Y@m@d@H/mpas_ics"' + value: 'set -xue ; rm -rf "{{ user.experiment_dir }}/@Y@m@d@H/mpas_ics" "{{ user.experiment_dir }}/@Y@m@d@H/ungrib_ics"' + account: "{{ platform.account }}" + partition: "{{ platform.service_partition }}" + join: + cyclestr: + value: '&LOGDIR;/{{ jobname }}_@Y@m@d@H.log' + walltime: "{{ scrubber.execution.walltime }}" + cores: !int "{{ scrubber.execution.cores }}" + dependency: + metataskdep_post: + attrs: + metatask: post + task_scrub_mpas_lbcs: + command: + cyclestr: + value: 'set -xue ; rm -rf "{{ user.experiment_dir }}/@Y@m@d@H/mpas_lbcs" "{{ user.experiment_dir }}/@Y@m@d@H/ungrib_lbcs"' account: "{{ platform.account }}" partition: "{{ platform.service_partition }}" join: From f0b29964c06f0c0bd2a0baaf1db29bb14c3512f9 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 18:53:06 +0000 Subject: [PATCH 2/9] the script that archives the lbcs --- scripts/archive_lbcs.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 scripts/archive_lbcs.sh diff --git a/scripts/archive_lbcs.sh b/scripts/archive_lbcs.sh new file mode 100755 index 0000000..4eb9edf --- /dev/null +++ b/scripts/archive_lbcs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -xue + +yyyymmddhh=$1 +archive_dir=$2 + +hsi mkdir -p "$archive_dir" || true + +archive="$archive_dir"/${yyyymmddhh}-lbc.tar + +listing=$( ls -1 $yyyymmddhh/mpas_lbcs/lbc.*nc ) + +htar -chpvf "$archive" $listing + +echo Normal completion. From 78be3c190b4c56aac91fab30d3e131a5e52e2169 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 18:53:27 +0000 Subject: [PATCH 3/9] some missing defaults --- ush/default_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ush/default_config.yaml b/ush/default_config.yaml index 4c9d1d4..a8f042e 100644 --- a/ush/default_config.yaml +++ b/ush/default_config.yaml @@ -35,6 +35,7 @@ user: workflow_blocks: - cold_start.yaml - post.yaml + hpss_archive_dir: /insert/a/valid/path/here data: # Known locations to mesh_files are updated from parm/machines/*.yaml files mesh_files: /path/to/nonstandard/location @@ -573,6 +574,10 @@ archiving: execution: walltime: 02:59:00 cores: 1 + lbcs: + execution: + walltime: 02:59:00 + cores: 1 scrubber: execution: From 412f7f7f3668a20e25509d9f3ab97e06ab36af5a Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 18:53:49 +0000 Subject: [PATCH 4/9] generate cron.sh file --- ush/experiment_gen.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ush/experiment_gen.py b/ush/experiment_gen.py index c6e4109..5e92b84 100755 --- a/ush/experiment_gen.py +++ b/ush/experiment_gen.py @@ -7,6 +7,8 @@ import argparse import inspect import logging +import os +import stat import sys from datetime import timedelta from pathlib import Path @@ -79,6 +81,7 @@ def main(): validated = validate(experiment_config.as_dict()) experiment_dir, experiment_file = setup_experiment_directory(validated) generate_workflow_files(experiment_config, experiment_file, mpas_app, user_config, validated) + make_cron_script(experiment_config, experiment_dir, mpas_app) stage_grid_files(experiment_config, experiment_dir) @@ -142,6 +145,21 @@ def required_nprocs(experiment_config: YAMLConfig) -> list[int]: return nprocs +def make_cron_script(experiment_config: YAMLConfig, experiment_dir: Path, mpas_app: Path) -> None: + filename = experiment_dir / "cron.sh" + machine = experiment_config["user"]["platform"] + logging.info("Creating CRON script: %s", filename) + with open(filename, 'wt') as fd: + fd.write(f"""#!/bin/sh +cd '{mpas_app}' +source ./load_wflow_modules.sh '{machine}' +cd '{experiment_dir}' +rocotorun -w rocoto.xml -d rocoto.db "$@" +""") + sb = os.stat(filename) + add_user_execute = sb.st_mode | stat.S_IXUSR + os.chmod(filename, add_user_execute) + def setup_experiment_directory(validated: Config) -> tuple[Path, Path]: """ Create the experiment directory and write experiment.yaml. From 0b84e37c9b2cec584ce623880bc757792c7aa7c5 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 18:54:21 +0000 Subject: [PATCH 5/9] MPAS HFIP 2026 resources, dates, and config --- ush/HFIP_2026.yaml | 20 +++++++++++ ush/workflows/hfip_2026.yaml | 26 +++++++++++--- ush/workflows/hfip_reservations.yaml | 54 ++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 ush/HFIP_2026.yaml create mode 100644 ush/workflows/hfip_reservations.yaml diff --git a/ush/HFIP_2026.yaml b/ush/HFIP_2026.yaml new file mode 100644 index 0000000..28a3664 --- /dev/null +++ b/ush/HFIP_2026.yaml @@ -0,0 +1,20 @@ +user: + platform: ursa + experiment_dir: /scratch5/purged/role.rtgsd-fv3-dev/MPAS_HFIP_2026/108_gfs_ops + hpss_archive_dir: /5year/BMC/fv3lam/role.rtgsd-fv3-dev/HFIP_MPAS_2026/108_gfs_ops + input_grib: /scratch5/purged/role.rtgsd-fv3-dev/MPAS_HFIP_2026/merged-input/102_download_lbcs/out + first_cycle: !!timestamp 2026-07-09T12:00:00 + last_cycle: !!timestamp 2026-12-31T12:00:00 + cycle_frequency: 24 + +forecast: + mpas: + length: 120 + +workflow: + attrs: + taskthrottle: 20 + realtime: true + +platform: + account: gsd-fv3-dev diff --git a/ush/workflows/hfip_2026.yaml b/ush/workflows/hfip_2026.yaml index 11c78fa..e374097 100644 --- a/ush/workflows/hfip_2026.yaml +++ b/ush/workflows/hfip_2026.yaml @@ -17,18 +17,28 @@ user: lbcs: data_stores: - disk - input_grib: /scratch3/BMC/gsd-fv3-dev/role.rtgsd-fv3-dev/MPAS_HFIP_2025/merged-input/out + input_grib: /scratch5/purged/role.rtgsd-fv3-dev/MPAS_HFIP_2026/merged-input/102_download_lbcs/out + data: mesh_files: /scratch3/BMC/wrfruc/Clark.Evans/MPAS-Model-v8.3.1/fix graphinfo_filename: "{{ user.mesh_label }}.graph.info" + platform: + hrrr_fix: /scratch3/BMC/wrfruc/Clark.Evans/MPAS-Model-v8.3.1/fix + data: gfs: - path: /scratch3/HFIP/hwrfv3/noscrub/hafs-input/COMGFSv17/gfs.{{ yyyymmdd }}/{{ hh }}/products/atmos/grib2/0p25/ + path: /scratch5/purged/role.rtgsd-fv3-dev/MPAS_HFIP_2026/merged-input/102_download_lbcs/out file_templates: - - gfs.t{{ hh }}z.pres_a.0p25.f{{ "%03d" | format(fcst_hr) }}.grib2 - - gfs.t{{ hh }}z.pres_b.0p25.f{{ "%03d" | format(fcst_hr) }}.grib2 - hrrr_fix: /scratch3/BMC/wrfruc/Clark.Evans/MPAS-Model-v8.3.1/fix + - '{{ cycle.strftime("%Y%m%d%H") }}/gfs.t{{ hh }}z.pgrb2.0p25.f{{ "%03d" | format(fcst_hr) }}' + - '{{ cycle.strftime("%Y%m%d%H") }}/gfs.t{{ hh }}z.pgrb2b.0p25.f{{ "%03d" | format(fcst_hr) }}' + +# data: +# gfs: +# path: /public/data/grids/gfs/0p25deg/grib2 +# file_templates: +# - '{{ cycle.strftime("%y%j%H00") }}{{ "%04d" % (lead_time.total_seconds() // 3600) }}' + workflow: tasks: task_get_ics_data: !remove @@ -39,6 +49,12 @@ workflow: value: cyclestr: value: '{{ user.input_grib }}/@Y@m@d@H/ready' + task_get_lbcs_data: + dependency: + datadep: + value: + cyclestr: + value: '{{ user.input_grib }}/@Y@m@d@H/ready' metatask_post: task_combine_grib_#fhr#: command: diff --git a/ush/workflows/hfip_reservations.yaml b/ush/workflows/hfip_reservations.yaml new file mode 100644 index 0000000..3872ea8 --- /dev/null +++ b/ush/workflows/hfip_reservations.yaml @@ -0,0 +1,54 @@ +platform: + account: rtgsd-fv3-dev +workflow: + tasks: + task_ungrib_ics: + account: rtgsd-fv3-dev + native: + cyclestr: + value: --reservation=gsd_ini_00 + task_ungrib_lbcs: + account: rtgsd-fv3-dev + native: + cyclestr: + value: --reservation=gsd_ini_00 + task_mpas_ics: + account: rtgsd-fv3-dev + native: + cyclestr: + value: --reservation=gsd_ini_00 + task_mpas_lbcs: + account: rtgsd-fv3-dev + native: + cyclestr: + value: --reservation=gsd_ini_00 + task_mpas: + attrs: + maxtries: 1 + account: rtgsd-fv3-dev + native: + cyclestr: + value: --reservation=gsd_fcst_00 + metatask_post: + task_mpassit_#fhr#: + account: rtgsd-fv3-dev + attrs: + maxtries: 9 + native: + cyclestr: + value: --reservation=gsd_post_00 + task_upp_#fhr#: + account: rtgsd-fv3-dev + native: + cyclestr: + value: --reservation=gsd_post_00 + task_combine_grib_#fhr#: + account: rtgsd-fv3-dev + native: + cyclestr: + value: --reservation=gsd_post_00 +forecast: + mpas: + execution: + batchargs: + walltime: 04:55:00 From 435baf495b436085e3708e1053cdb1aecbaf2ed9 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 18:54:45 +0000 Subject: [PATCH 6/9] a wrapper around experiment_gen.py that lists MPAS HFIP 2026 yaml files --- ush/generate_MPAS_HFIP_2026_experiment.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 ush/generate_MPAS_HFIP_2026_experiment.sh diff --git a/ush/generate_MPAS_HFIP_2026_experiment.sh b/ush/generate_MPAS_HFIP_2026_experiment.sh new file mode 100755 index 0000000..d82c9ea --- /dev/null +++ b/ush/generate_MPAS_HFIP_2026_experiment.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -xue +cd .. +source ./load_wflow_modules.sh ursa +cd ush +./experiment_gen.py workflows/hfip_2026.yaml workflows/hfip_resources.yaml workflows/hfip_reservations.yaml HFIP_2026.yaml From 5fe14963ba68d16f773dbbccfea62210bae20688 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 19:57:46 +0000 Subject: [PATCH 7/9] revise cron.sh to have module command --- ush/experiment_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/experiment_gen.py b/ush/experiment_gen.py index 5e92b84..5b18ec0 100755 --- a/ush/experiment_gen.py +++ b/ush/experiment_gen.py @@ -150,7 +150,7 @@ def make_cron_script(experiment_config: YAMLConfig, experiment_dir: Path, mpas_a machine = experiment_config["user"]["platform"] logging.info("Creating CRON script: %s", filename) with open(filename, 'wt') as fd: - fd.write(f"""#!/bin/sh + fd.write(f"""#!/bin/bash --login cd '{mpas_app}' source ./load_wflow_modules.sh '{machine}' cd '{experiment_dir}' From f7d25495c8659fd6fd525e6763cd4b33a92b1198 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 20:28:27 +0000 Subject: [PATCH 8/9] make code check happier --- ush/experiment_gen.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/experiment_gen.py b/ush/experiment_gen.py index 5b18ec0..fdc2f0d 100755 --- a/ush/experiment_gen.py +++ b/ush/experiment_gen.py @@ -146,19 +146,19 @@ def required_nprocs(experiment_config: YAMLConfig) -> list[int]: def make_cron_script(experiment_config: YAMLConfig, experiment_dir: Path, mpas_app: Path) -> None: - filename = experiment_dir / "cron.sh" + cron_sh = experiment_dir / "cron.sh" machine = experiment_config["user"]["platform"] - logging.info("Creating CRON script: %s", filename) - with open(filename, 'wt') as fd: + logging.info("Creating CRON script: %s", cron_sh) + with cron_sh.open('w') as fd: fd.write(f"""#!/bin/bash --login cd '{mpas_app}' source ./load_wflow_modules.sh '{machine}' cd '{experiment_dir}' rocotorun -w rocoto.xml -d rocoto.db "$@" """) - sb = os.stat(filename) + sb = cron_sh.stat() add_user_execute = sb.st_mode | stat.S_IXUSR - os.chmod(filename, add_user_execute) + cron_sh.chmod(add_user_execute) def setup_experiment_directory(validated: Config) -> tuple[Path, Path]: """ From d604e6812165cff0688e5785b85afb73e783bd13 Mon Sep 17 00:00:00 2001 From: "Samuel.Trahan" Date: Thu, 9 Jul 2026 20:31:31 +0000 Subject: [PATCH 9/9] make code check happier --- ush/experiment_gen.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/experiment_gen.py b/ush/experiment_gen.py index fdc2f0d..1c6cdd3 100755 --- a/ush/experiment_gen.py +++ b/ush/experiment_gen.py @@ -7,7 +7,6 @@ import argparse import inspect import logging -import os import stat import sys from datetime import timedelta @@ -149,7 +148,7 @@ def make_cron_script(experiment_config: YAMLConfig, experiment_dir: Path, mpas_a cron_sh = experiment_dir / "cron.sh" machine = experiment_config["user"]["platform"] logging.info("Creating CRON script: %s", cron_sh) - with cron_sh.open('w') as fd: + with cron_sh.open("w") as fd: fd.write(f"""#!/bin/bash --login cd '{mpas_app}' source ./load_wflow_modules.sh '{machine}'