diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9e8c35db..937cd71f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,7 +14,7 @@ jobs: name: Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Prepare Environment run: make devenv - name: Code-Quality Check diff --git a/build.sh b/build.sh index 237b0468..58ed54eb 100755 --- a/build.sh +++ b/build.sh @@ -107,7 +107,16 @@ install_mpas () { make $target CORE=$CORE $opts mkdir -pv $EXEC_DIR cp -v ${CORE}_model $EXEC_DIR - test $CORE == atmosphere && ./build_tables_tempo || true + if [[ $CORE == atmosphere ]] ; then + ./build_tables + if [[ -x ./build_tables_tempo ]] ; then + ./build_tables_tempo + else + curl -sSLO https://github.com/ufs-community/MPAS-Model/releases/download/MPAS-v8.3.1-2.14/tempo_data.tar + tar -xvf tempo_data.tar + rm -v tempo_data.tar + fi + fi ) } diff --git a/environment.yml b/environment.yml index 803239c1..eca51554 100644 --- a/environment.yml +++ b/environment.yml @@ -3,6 +3,7 @@ channels: - conda-forge - ufs-community dependencies: + - python==3.13.* - metis==5.1.0.* - pydantic==2.11.* - - uwtools==2.9.* + - uwtools==2.16.* diff --git a/load_wflow_modules.sh b/load_wflow_modules.sh index a7b6dc48..707d73b6 100644 --- a/load_wflow_modules.sh +++ b/load_wflow_modules.sh @@ -1,3 +1,9 @@ +machine=$1 + +if [[ -z "$machine" ]]; then + echo "usage: source load_wflow_modules.sh [machine]" + return +fi scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) diff --git a/parm/machines/hera.yaml b/parm/machines/hera.yaml index 70f86ef1..10bea9ed 100644 --- a/parm/machines/hera.yaml +++ b/parm/machines/hera.yaml @@ -9,6 +9,8 @@ forecast: execution: batchargs: partition: hera + nodes: 20 + tasks_per_node: 40 post: mpassit: execution: diff --git a/parm/machines/jet.yaml b/parm/machines/jet.yaml deleted file mode 100644 index e9f18cea..00000000 --- a/parm/machines/jet.yaml +++ /dev/null @@ -1,32 +0,0 @@ -data: - mesh_files: "{{ platform.hrrr_fix }}" -platform: - scheduler: slurm - crtm_dir: /misc/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/intel/2021.5.0/crtm-fix-2.4.0.1_emc-bscege4/fix - geog_data: - global: /lfs5/BMC/wrfruc/Michael.Barlage/mpas/data/mpas_static - raphrrr: /mnt/lfs5/BMC/wrfruc/HRRRv5/geog - hrrr_fix: /misc/whome/role.wrfruc/HRRRv5/fix - data: - gfs: - path: /public/data/grids/gfs/0p25deg/grib2 - file_templates: '@y@j@H00{{ "%04d" % fcst_hr }}' -forecast: - mpas: - execution: - batchargs: - partition: kjet,xjet -post: - mpassit: - fix_dir: /misc/whome/role.wrfruc/HRRRv5/fix/MPASSIT/ - execution: - batchargs: - partition: kjet,xjet - upp: - execution: - batchargs: - partition: kjet,xjet -graphics: - execution: - batchargs: - partition: kjet diff --git a/parm/machines/ursa.yaml b/parm/machines/ursa.yaml index 3282ee94..8e0af808 100644 --- a/parm/machines/ursa.yaml +++ b/parm/machines/ursa.yaml @@ -1,8 +1,8 @@ data: - mesh_files: /scratch3/BMC/wrfruc/cholt/MPAS-Model/fix + mesh_files: /scratch3/BMC/rtwrfruc/RRFSv2X/code/rrfs-workflow.20260603/fix/{{ user.mesh_label }} platform: crtm_dir: /contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.2.1/install/oneapi/2024.2.1/crtm-fix-2.4.0.1_emc-t5jym6r/fix - hrrr_fix: /scratch3/BMC/wrfruc/cholt/MPAS-Model/fix + hrrr_fix: /scratch3/BMC/rtwrfruc/RRFSv2X/code/rrfs-workflow.20260603/fix scheduler: slurm geog_data: global: /scratch2/BMC/wrfruc/talcott/mpas/global/static/geog/ @@ -10,30 +10,26 @@ platform: data: gfs: path: /public/data/grids/gfs/0p25deg/grib2 - file_templates: '@y@j@H00{{ "%04d" % fcst_hr }}' + file_templates: '@y@j@H00{forecast_hr:04d}' create_ics: mpas_init: execution: - envcmds: + envcmds: &ursa_intel_mpi - ulimit -s unlimited || true - module use {{ user.mpas_app }}/modulefiles - module load build_{{ user.platform }}_intel_ifort + - module load impi-collective-settings/1.0.0 + - export I_MPI_COLL_INTRANODE=pt2pt create_lbcs: mpas_init: execution: - envcmds: - - ulimit -s unlimited || true - - module use {{ user.mpas_app }}/modulefiles - - module load build_{{ user.platform }}_intel_ifort + envcmds: *ursa_intel_mpi forecast: mpas: execution: batchargs: partition: u1-compute - envcmds: - - ulimit -s unlimited || true - - module use {{ user.mpas_app }}/modulefiles - - module load build_{{ user.platform }}_intel_ifort + envcmds: *ursa_intel_mpi post: mpassit: execution: diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.analysis b/parm/mpas/hrrrv5/stream_list.atmosphere.analysis new file mode 100644 index 00000000..0f4c7efc --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.analysis @@ -0,0 +1,10 @@ +pressure_p +rho +scalars +surface_pressure +theta +u +uReconstructZonal +uReconstructMeridional +refl10cm +w diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.background b/parm/mpas/hrrrv5/stream_list.atmosphere.background new file mode 100644 index 00000000..3ef94426 --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.background @@ -0,0 +1,27 @@ +cldfrac +lai +pressure_base +pressure_p +re_cloud +re_ice +re_snow +rho +scalars +skintemp +smois +snowc +snowh +surface_pressure +theta +tslb +u +u10 +uReconstructZonal +uReconstructMeridional +v10 +vegfra +w +xice +refl10cm +t2m +q2 diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.control b/parm/mpas/hrrrv5/stream_list.atmosphere.control new file mode 100644 index 00000000..17b554e7 --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.control @@ -0,0 +1,10 @@ +scalars +spechum +stream_function +surface_pressure +temperature +uReconstructZonal +uReconstructMeridional +velocity_potential +refl10cm +w diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.da_state b/parm/mpas/hrrrv5/stream_list.atmosphere.da_state new file mode 100644 index 00000000..b5c70cf6 --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.da_state @@ -0,0 +1,293 @@ +scalars +initial_time +xtime +Time +u +w +rho_zz +theta_m +rho +pressure_p +pressure_base +theta +relhum +rho_base +theta_base +uReconstructZonal +uReconstructMeridional +circulation +exner +exner_base +rtheta_base +ru +ru_p +rw +rw_p +rtheta_p +rho_p +surface_pressure +rainprod +evapprod +isltyp +soilf +ivgtyp +landusef +mminlu +landmask +shdmin +shdmax +snoalb +albedo12m +greenfrac +lai12m +cldfrac +re_cloud +re_ice +re_snow +nwfa2d +nifa2d +nt_c +mu_c +tend_sfc_pressure +rt_diabatic_tend +nsteps_accum +ndays_accum +tlag +tday_accum +tyear_mean +tyear_accum +refl10cm +refl10cm_max +rainc +i_rainnc +rainncv +snowncv +snownc +graupelncv +graupelnc +sr +i_rainc +cuprec +raincv +nca +cubot +cutop +w0avg +qcg +sh3d +sm3d +qke +qke_adv +cov +qsq +tsq +tke_pbl +wstar +delta +el_pbl +edmf_a +edmf_w +edmf_qt +edmf_qc +edmf_thl +edmf_ent +sub_thl +sub_qv +det_thl +det_qv +cldfrac_bl +qc_bl +qi_bl +ch +kpbl +hpbl +hfx +cldfrac_cu +refl10cm_cu +mol +qfx +qsfc +ustm +zol +br +cd +cda +chs +chs2 +cqs2 +cqs +ck +cka +cpm +flhc +flqc +gz1oz0 +lh +psim +psih +qgh +regime +rmol +wspd +fh +fm +th2m +dusfcg +dvsfcg +dtaux3d +dtauy3d +rubldiff +rvbldiff +i_acswdnb +i_acswdnbc +i_acswdnt +i_acswdntc +i_acswupb +i_acswupbc +i_acswupt +i_acswuptc +acswdnb +acswdnbc +acswdnt +acswdntc +acswupb +acswupbc +acswupt +acswuptc +gsw +i_aclwdnb +i_aclwdnbc +i_aclwdnt +i_aclwdntc +i_aclwupb +i_aclwupbc +i_aclwupt +i_aclwuptc +aclwdnb +aclwdnbc +aclwdnt +aclwdntc +aclwupb +aclwupbc +aclwupt +aclwuptc +glw +o3clim +o3vmr +acsnom +acsnow +canwat +chklowq +grdflx +noahres +potevp +sfcrunoff +smstav +smstot +snopcx +snotime +sstsk +sstsk_dtc +sstsk_dtw +udrunoff +rhosnf +precipfr +qsg +qvg +dew +soilt1 +tsnav +acrunoff +snowfallac +smfr3d +k22_shallow +kbcon_shallow +ktop_shallow +ktop_deep +xmb_total +xmb_shallow +qc_cu +qi_cu +rthmpten +rqvmpten +rqcmpten +rqrmpten +rqimpten +rqsmpten +rqgmpten +rncmpten +rnimpten +rnrmpten +rnifampten +rnwfampten +rthcuten +rqvcuten +rqccuten +rqicuten +rqrcuten +rqscuten +rqvdynten +rthdynten +rucuten +rvcuten +rublten +rvblten +rthblten +rqvblten +rqcblten +rqiblten +rqsblten +rncblten +rniblten +rnifablten +rnwfablten +rthratensw +rthratenlw +vcpool +sub3d_rthcuten +sub3d_rqvcuten +sub3d_rucuten +sub3d_rvcuten +sigma_deep +smcrel +t_oml +t_oml_initial +t_oml_200m_initial +h_oml +hu_oml +hv_oml +lai +isice_lu +iswater_lu +sfc_albbck +sfc_albedo +sfc_emibck +mavail +sfc_emiss +thc +ust +xicem +z0 +znt +skintemp +snow +snowc +snowh +sst +tmn +vegfra +seaice +xice +xland +u10 +v10 +q2 +t2m +precipw +dzs +zs +ter +sh2o +smois +tslb +h_oml_initial diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.diag_ugwp b/parm/mpas/hrrrv5/stream_list.atmosphere.diag_ugwp new file mode 100644 index 00000000..e56c26bb --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.diag_ugwp @@ -0,0 +1,22 @@ +initial_time +xtime +Time +dtaux3d_ls +dtauy3d_ls +dtaux3d_bl +dtauy3d_bl +dtaux3d_ss +dtauy3d_ss +dtaux3d_fd +dtauy3d_fd +dusfc_ls +dvsfc_ls +dusfc_bl +dvsfc_bl +dusfc_ss +dvsfc_ss +dusfc_fd +dvsfc_fd +dudt_ngw +dvdt_ngw +dtdt_ngw diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.diagnostics b/parm/mpas/hrrrv5/stream_list.atmosphere.diagnostics new file mode 100644 index 00000000..a289db9a --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.diagnostics @@ -0,0 +1,146 @@ +initial_time +xtime +Time +snowncv +rainncv +graupelncv +prec_acc_c +prec_acc_nc +snow_acc_nc +mslp +relhum_50hPa +relhum_100hPa +relhum_200hPa +relhum_250hPa +relhum_500hPa +relhum_700hPa +relhum_850hPa +relhum_925hPa +dewpoint_50hPa +dewpoint_100hPa +dewpoint_200hPa +dewpoint_250hPa +dewpoint_500hPa +dewpoint_700hPa +dewpoint_850hPa +dewpoint_925hPa +temperature_50hPa +temperature_100hPa +temperature_200hPa +temperature_250hPa +temperature_500hPa +temperature_700hPa +temperature_850hPa +temperature_925hPa +height_50hPa +height_100hPa +height_200hPa +height_250hPa +height_500hPa +height_700hPa +height_850hPa +height_925hPa +uzonal_50hPa +uzonal_100hPa +uzonal_200hPa +uzonal_250hPa +uzonal_500hPa +uzonal_700hPa +uzonal_850hPa +uzonal_925hPa +umeridional_50hPa +umeridional_100hPa +umeridional_200hPa +umeridional_250hPa +umeridional_500hPa +umeridional_700hPa +umeridional_850hPa +umeridional_925hPa +w_50hPa +w_100hPa +w_200hPa +w_250hPa +w_500hPa +w_700hPa +w_850hPa +w_925hPa +vorticity_50hPa +vorticity_100hPa +vorticity_200hPa +vorticity_250hPa +vorticity_500hPa +vorticity_700hPa +vorticity_850hPa +vorticity_925hPa +t_isobaric +t_iso_levels +z_isobaric +z_iso_levels +meanT_500_300 +olrtoa +rainc +rainnc +frainnc +refl10cm +refl10cm_max +refl10cm_1km +refl10cm_1km_max +precipw +u10 +v10 +q2 +t2m +th2m +cape +cin +lcl +lfc +srh_0_1km +srh_0_3km +uzonal_surface +uzonal_1km +uzonal_6km +umeridional_surface +umeridional_1km +umeridional_6km +temperature_surface +dewpoint_surface +updraft_helicity_max +w_velocity_max +w_velocity_min +w_velocity_mean +wind_speed_level1_max +t_oml +t_oml_initial +t_oml_200m_initial +h_oml +h_oml_initial +hu_oml +hv_oml +updraft_helicity_max_01 +updraft_helicity_max_16 +updraft_helicity_max0_15 +updraft_helicity_max05_15 +updraft_helicity_max_02 +updraft_helicity_max_03 +updraft_helicity_min_01 +updraft_helicity_min_16 +updraft_helicity_min0_15 +updraft_helicity_min05_15 +updraft_helicity_min_02 +updraft_helicity_min_03 +wind02_max +wind02_integral_max +wind_speed_10m_max +wind_speed_10m_mean +u10m_mean +v10m_mean +swdnb +swupt +swddir +swddni +swddif +max_hail_diameter_sfc +max_hail_diameter_sfc_acc +max_hail_diameter_column +max_hail_diameter_column_acc diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.ensemble b/parm/mpas/hrrrv5/stream_list.atmosphere.ensemble new file mode 100644 index 00000000..98ad1dc4 --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.ensemble @@ -0,0 +1,9 @@ +pressure_base +pressure_p +scalars +surface_pressure +theta +uReconstructZonal +uReconstructMeridional +refl10cm +w diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.output b/parm/mpas/hrrrv5/stream_list.atmosphere.output new file mode 100644 index 00000000..b7f86ccd --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.output @@ -0,0 +1,45 @@ +scalars +initial_time +xtime +Time +zgrid +w +pressure +surface_pressure +rho +theta +uReconstructZonal +uReconstructMeridional +hpbl +hfx +lh +xland +skintemp +snow +snowh +sst +vegfra +shdmin +shdmax +canwat +sh2o +smois +tslb +snowfallac +sfcrunoff +ivgtyp +isltyp +sfc_albedo +cldfrac_bl +snowc +snownc +graupelnc +dzs +zs +ter +znt +qke +el_pbl +ust +br +rmol diff --git a/parm/mpas/hrrrv5/stream_list.atmosphere.surface b/parm/mpas/hrrrv5/stream_list.atmosphere.surface new file mode 100644 index 00000000..db54cab4 --- /dev/null +++ b/parm/mpas/hrrrv5/stream_list.atmosphere.surface @@ -0,0 +1,2 @@ +sst +xice diff --git a/parm/mpas/stream_list.atmosphere.diag_ugwp b/parm/mpas/stream_list.atmosphere.diag_ugwp deleted file mode 120000 index d656d714..00000000 --- a/parm/mpas/stream_list.atmosphere.diag_ugwp +++ /dev/null @@ -1 +0,0 @@ -../../src/MPAS-Model/stream_list.atmosphere.diag_ugwp \ No newline at end of file diff --git a/parm/mpas/stream_list.atmosphere.diagnostics b/parm/mpas/stream_list.atmosphere.diagnostics deleted file mode 120000 index 360c5407..00000000 --- a/parm/mpas/stream_list.atmosphere.diagnostics +++ /dev/null @@ -1 +0,0 @@ -../../src/MPAS-Model/stream_list.atmosphere.diagnostics \ No newline at end of file diff --git a/parm/mpas/stream_list.atmosphere.output b/parm/mpas/stream_list.atmosphere.output deleted file mode 120000 index ef125906..00000000 --- a/parm/mpas/stream_list.atmosphere.output +++ /dev/null @@ -1 +0,0 @@ -../../src/MPAS-Model/stream_list.atmosphere.output \ No newline at end of file diff --git a/parm/mpas/stream_list.atmosphere.surface b/parm/mpas/stream_list.atmosphere.surface deleted file mode 120000 index 534f2266..00000000 --- a/parm/mpas/stream_list.atmosphere.surface +++ /dev/null @@ -1 +0,0 @@ -../../src/MPAS-Model/stream_list.atmosphere.surface \ No newline at end of file diff --git a/parm/wflow/cold_start.yaml b/parm/wflow/cold_start.yaml index 6b0d5dcc..e6157894 100644 --- a/parm/wflow/cold_start.yaml +++ b/parm/wflow/cold_start.yaml @@ -152,7 +152,7 @@ workflow: cyclestr: value: '&LOGDIR;/{{ jobname }}_@Y@m@d@H.log' walltime: "{{ forecast.mpas.execution.batchargs.walltime }}" - cores: !int "{{ forecast.mpas.execution.batchargs.cores }}" + nodes: "{{ forecast.mpas.execution.batchargs.nodes }}:ppn={{ forecast.mpas.execution.batchargs.tasks_per_node }}" partition: "{{ forecast.mpas.execution.batchargs.get('partition') }}" dependency: taskdep: diff --git a/parm/wflow/post.yaml b/parm/wflow/post.yaml index 7f00e80c..4d3412d8 100644 --- a/parm/wflow/post.yaml +++ b/parm/wflow/post.yaml @@ -30,8 +30,7 @@ workflow: value: '&LOGDIR;/{{ jobname }}_@Y@m@d@H.log' partition: "{{ post.mpassit.execution.batchargs.partition }}" walltime: "{{ post.mpassit.execution.batchargs.walltime }}" - native: "--nodes={{ post.mpassit.execution.batchargs.nodes }}" - cores: !int "{{ post.mpassit.execution.batchargs.cores }}" + nodes: "{{ post.mpassit.execution.batchargs.nodes }}:ppn={{ post.mpassit.execution.batchargs.tasks_per_node }}" dependency: or: taskdep: diff --git a/scripts/common.py b/scripts/common.py index de0dcbd1..853155a6 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -52,7 +52,7 @@ def run_component( logging.info("Running %s in %s", driver_class.__name__, rundir) task = driver.run() if not task.ready: - logging.error("Error occurred. Expected file %s not found.", task.refs[0]) + logging.error("Error occurred. Expected file %s not found.", task.ref[0]) sys.exit(1) return driver diff --git a/scripts/ungrib.py b/scripts/ungrib.py index c91e37fc..8d9d14a3 100755 --- a/scripts/ungrib.py +++ b/scripts/ungrib.py @@ -11,7 +11,7 @@ sys.path.append(str(Path(__file__).parent.parent)) -from iotaa import asset, external, task, tasks +from iotaa import Asset, collection, external, task from uwtools.api.config import get_yaml_config from uwtools.api.logging import use_uwtools_logger from uwtools.api.ungrib import Ungrib @@ -23,7 +23,7 @@ @external def file(path: Path): yield f"{path}" - yield asset(path, path.is_file) + yield Asset(path, path.is_file) @task @@ -37,7 +37,7 @@ def regrid_input(driver: Ungrib, infile: Path, wgrib_config: dict): taskname = f"wgrib2 regrid {infile}" yield taskname outfile = driver.rundir / f"tmp.{infile.name}.grib2" - yield asset(outfile, outfile.is_file) + yield Asset(outfile, outfile.is_file) yield driver.gribfiles() gribfile = infile.resolve() # Removes the GRIBFILE.* link. @@ -76,7 +76,7 @@ def merge_vector_fields(driver: Ungrib, infile: Path, wgrib_config: dict): taskname = f"wgrib2 merge vector fields {infile}" yield taskname outfile = driver.rundir / f"tmp2.{infile.name}.grib2" - yield asset(outfile, outfile.is_file) + yield Asset(outfile, outfile.is_file) regrid_task = regrid_input(driver, infile, wgrib_config) yield regrid_task options = [ @@ -102,7 +102,11 @@ def merge_vector_fields(driver: Ungrib, infile: Path, wgrib_config: dict): outfile.unlink() -@tasks +def regrid(driver: Ungrib, wgrib2_config: dict): + regrid_all(driver, wgrib2_config) + + +@collection def regrid_all(driver: Ungrib, wgrib2_config: dict): """ Use wgrib2 to regrid the winds. @@ -130,16 +134,11 @@ def run_ungrib(config_file, cycle, key_path): lbcs_summary = get_yaml_config(rundir / "LBCS.yaml") gribfiles.extend(Path(rundir, p) for p in lbcs_summary) ungrib_block["ungrib"]["gribfiles"] = [str(p) for p in gribfiles] - driver = Ungrib(config=expt_config, cycle=cycle, key_path=key_path) - yield [asset(x, x.is_file) for x in driver.output["paths"]] - yield ( - regrid_all(driver, walk_key_path(config=expt_config, key_path=key_path)["wgrib2"]) - if external_model == "RRFS" - else None - ) - # Run ungrib. - logging.info("Running %s in %s", Ungrib.__name__, driver.rundir) - driver.run() + driver = Ungrib(config=ungrib_block, cycle=cycle) + yield [Asset(x, x.is_file) for x in driver.output["paths"]] + if external_model == "RRFS": + regrid(driver, ungrib_block["wgrib2"]) + yield driver.run() def main(): diff --git a/src/MPAS-Model b/src/MPAS-Model index a3391257..13aef82a 160000 --- a/src/MPAS-Model +++ b/src/MPAS-Model @@ -1 +1 @@ -Subproject commit a3391257ab2ec4f2541fa7abffeb49f6c55bc961 +Subproject commit 13aef82a0c0356977df26b5a32da82b8f585d26f diff --git a/src/MPASSIT b/src/MPASSIT index 2144f94e..9e658dbe 160000 --- a/src/MPASSIT +++ b/src/MPASSIT @@ -1 +1 @@ -Subproject commit 2144f94ecfa144d6032b29f8fff60fec81690316 +Subproject commit 9e658dbe52b682613fb60c16fa2a7d67e8a8efe2 diff --git a/tests/scripts/test_common.py b/tests/scripts/test_common.py index 39c3fcf3..2f378a0b 100644 --- a/tests/scripts/test_common.py +++ b/tests/scripts/test_common.py @@ -89,7 +89,7 @@ def test_run_component_failure(caplog, test_driver, args): caplog.set_level("ERROR") driver = test_driver with ( - patch.object(driver, "run", return_value=Mock(ready=False, refs=["/some/rundir/file"])), + patch.object(driver, "run", return_value=Mock(ready=False, ref=["/some/rundir/file"])), patch("scripts.common.use_uwtools_logger"), patch("sys.exit") as sysexit, ): diff --git a/tests/scripts/test_ungrib.py b/tests/scripts/test_ungrib.py index 42817e84..80d003fb 100644 --- a/tests/scripts/test_ungrib.py +++ b/tests/scripts/test_ungrib.py @@ -53,13 +53,13 @@ def ungrib_driver(ungrib_config): @iotaa.external def noop(*_args, **_kwargs): yield "No op" - yield iotaa.asset(None, lambda: True) + yield iotaa.Asset(None, lambda: True) @iotaa.external def noop_not_ready(*_args, **_kwargs): yield "No op not ready" - yield iotaa.asset(None, lambda: False) + yield iotaa.Asset(None, lambda: False) def test_file__missing(tmp_path): @@ -128,9 +128,10 @@ def test_regrid_input(ungrib_driver, tmp_path): grib_file.touch() infile = tmp_path / "GRIBFILE.AAA" infile.symlink_to(grib_file) + with ( patch.object(ungrib, "run_shell_cmd") as run_shell_cmd, - patch.object(ungrib.Ungrib, "gribfiles") as gribfiles, + patch.object(ungrib.Ungrib, "gribfiles", side_effect=noop) as gribfiles, ): ungrib.regrid_input(ungrib_driver, infile, wgrib_config) gribfiles.assert_called_once() @@ -153,7 +154,7 @@ def test_regrid_all(tmp_path, ungrib_driver): gribfiles.ref = [tmp_path / f"GRIBFILE.{label}" for label in ("AAA", "AAB", "AAC", "AAD")] with ( patch.object(ungrib.Ungrib, "gribfiles", return_value=gribfiles), - patch.object(ungrib, "merge_vector_fields") as wgrib_task, + patch.object(ungrib, "merge_vector_fields", side_effect=noop) as wgrib_task, ): ungrib.regrid_all(ungrib_driver, {"wgrib2": {}}) assert expected_calls == wgrib_task.call_args_list @@ -170,7 +171,7 @@ def test_run_ungrib_gfs(outcome, tmp_path, ungrib_config): ungrib_config.dump(config_file) cycle = datetime(2025, 1, 1, 12, tzinfo=timezone.utc) side_effect = (tmp_path / "runscript.ungrib.done").touch() if outcome == "pass" else None - with patch.object(ungrib.Ungrib, "run", side_effect=side_effect) as run: + with patch.object(ungrib.Ungrib, "run", side_effect=side_effect, wraps=noop) as run: task_state = ungrib.run_ungrib(config_file, cycle, ["ungrib_ics"]) if outcome == "pass": assert (tmp_path / "runscript.ungrib.done").exists() @@ -192,7 +193,7 @@ def test_run_ungrib_rrfs_ics(tmp_path, ungrib_config): cycle = datetime(2025, 1, 1, 12, tzinfo=timezone.utc) with ( - patch.object(ungrib.Ungrib, "run") as run, + patch.object(ungrib.Ungrib, "run", wraps=noop) as run, patch.object(ungrib, "regrid_all", wraps=noop) as regrid_all, ): ungrib.run_ungrib(config_file, cycle, ["ungrib_ics"]) @@ -213,7 +214,7 @@ def test_run_ungrib_rrfs_lbcs(tmp_path, ungrib_config): cycle = datetime(2025, 1, 1, 12, tzinfo=timezone.utc) with ( - patch.object(ungrib.Ungrib, "run") as run, + patch.object(ungrib.Ungrib, "run", wraps=noop) as run, patch.object(ungrib, "regrid_all", wraps=noop) as regrid_all, ): ungrib.run_ungrib(config_file, cycle, ["ungrib_lbcs"]) diff --git a/tests/ush/test_experiment_gen.py b/tests/ush/test_experiment_gen.py index a75cf75a..413c1727 100644 --- a/tests/ush/test_experiment_gen.py +++ b/tests/ush/test_experiment_gen.py @@ -20,7 +20,10 @@ def test_config(tmp_path): "ics": {"external_model": "GFS"}, "lbcs": {"external_model": "GFS"}, }, - "data": {"mesh_files": str(tmp_path / "meshes")}, + "data": { + "mesh_files": str(tmp_path / "meshes"), + "graphinfo_filename": "testmesh.graph.info", + }, "create_ics": {"mpas_init": {"execution": {"batchargs": {"cores": 32}}}}, "forecast": {"mpas": {"execution": {"batchargs": {"nodes": 2, "tasks_per_node": 32}}}}, "workflow": { @@ -241,9 +244,7 @@ def test_stage_grid_files(test_config, validated_config): existing_part = validated_config.user.experiment_dir / f"{mesh_file.name}.part.32" existing_part.write_text("partitioned content") with patch.object(experiment_gen, "create_grid_files") as create: - experiment_gen.stage_grid_files( - test_config, validated_config.user.experiment_dir, validated_config - ) + experiment_gen.stage_grid_files(test_config, validated_config.user.experiment_dir) create.assert_called_once_with(validated_config.user.experiment_dir, mesh_file, 64) diff --git a/ush/default_config.yaml b/ush/default_config.yaml index ddd5c64e..7a269700 100644 --- a/ush/default_config.yaml +++ b/ush/default_config.yaml @@ -28,7 +28,7 @@ user: forecast: output_interval: 6 length: !timedelta "{{ forecast.mpas.length }}" - mesh_label: hrrrv5 + mesh_label: conus3km platform: big_computer workflow_blocks: - cold_start.yaml @@ -36,6 +36,7 @@ user: data: # Known locations to mesh_files are updated from parm/machines/*.yaml files mesh_files: /path/to/nonstandard/location + graphinfo_filename: "graphinfo/{{ user.mesh_label }}.graph.info" platform: # Supported scheduler options include Slurm, PBSPro, and LSF scheduler: slurm @@ -120,9 +121,9 @@ create_ics: files_to_copy: &mpas_init_files_to_copy "{{ user.mesh_label }}.static.nc": "{{ data.mesh_files }}/{{ user.mesh_label }}.static.nc" "{{ user.mesh_label }}.graph.info.part.{{create_ics.mpas_init.execution.batchargs.cores }}": '{{ user.experiment_dir }}/{{ user.mesh_label }}.graph.info.part.{{ create_ics.mpas_init.execution.batchargs.cores }}' - stream_list.atmosphere.diagnostics: "{{ user.mpas_app }}/parm/mpas/stream_list.atmosphere.diagnostics" - stream_list.atmosphere.output: "{{ user.mpas_app }}/parm/mpas/stream_list.atmosphere.output" - stream_list.atmosphere.surface: "{{ user.mpas_app }}/parm/mpas/stream_list.atmosphere.surface" + stream_list.atmosphere.diagnostics: "{{ user.mpas_app }}/parm/mpas/hrrrv5/stream_list.atmosphere.diagnostics" + stream_list.atmosphere.output: "{{ user.mpas_app }}/parm/mpas/hrrrv5/stream_list.atmosphere.output" + stream_list.atmosphere.surface: "{{ user.mpas_app }}/parm/mpas/hrrrv5/stream_list.atmosphere.surface" files_to_link: &mpas_init_files_to_link CAM_ABS_DATA.DBL: "{{ user.mpas_app }}/src/MPAS-Model/CAM_ABS_DATA.DBL" CAM_AEROPT_DATA.DBL: "{{ user.mpas_app }}/src/MPAS-Model/CAM_AEROPT_DATA.DBL" @@ -137,7 +138,6 @@ create_ics: RRTMG_SW_DATA.DBL: "{{ user.mpas_app }}/src/MPAS-Model/RRTMG_SW_DATA.DBL" SOILPARM.TBL: "{{ user.mpas_app }}/src/MPAS-Model/SOILPARM.TBL" VEGPARM.TBL: "{{ user.mpas_app }}/src/MPAS-Model/VEGPARM.TBL" - QNWFA_QNIFA_SIGMA_MONTHLY.dat: "{{ data.mesh_files }}/QNWFA_QNIFA_SIGMA_MONTHLY.dat" namelist: base_file: "{{ user.mpas_app }}/src/MPAS-Model/namelist.init_atmosphere" update_values: &mpas_init_update_values @@ -154,7 +154,7 @@ create_ics: vertical_grid: config_blend_bdy_terrain: true config_ztop: 25878.712 - config_specified_zeta_levels: '{{ platform.hrrr_fix }}/L60.txt' + config_specified_zeta_levels: '{{ platform.hrrr_fix }}/vert_levels/L60.txt' preproc_stages: config_native_gwd_static: false config_static_interp: false @@ -229,7 +229,9 @@ forecast: domain: regional execution: batchargs: - cores: 800 + nodes: 5 + tasks_per_node: 160 + cores: !int "{{ forecast.mpas.execution.batchargs.tasks_per_node * forecast.mpas.execution.batchargs.nodes }}" walltime: 02:30:00 envcmds: - ulimit -s unlimited || true @@ -240,20 +242,20 @@ forecast: - "--ntasks={{ forecast.mpas.execution.batchargs.cores }}" mpicmd: srun files_to_copy: + "{{ user.mesh_label }}.graph.info.part.{{ forecast.mpas.execution.batchargs.cores }}": '{{ user.experiment_dir }}/{{ user.mesh_label }}.graph.info.part.{{ forecast.mpas.execution.batchargs.cores }}' + stream_list.atmosphere.diagnostics: "{{ user.mpas_app }}/parm/mpas/hrrrv5/stream_list.atmosphere.diagnostics" + stream_list.atmosphere.output: "{{ user.mpas_app }}/parm/mpas/hrrrv5/stream_list.atmosphere.output" + stream_list.atmosphere.surface: "{{ user.mpas_app }}/parm/mpas/hrrrv5/stream_list.atmosphere.surface" + files_to_hardlink: "{{ user.mesh_label }}.init.nc": "{{ create_ics.mpas_init.rundir }}/{{ user.mesh_label }}.init.nc" - "{{ user.mesh_label }}.graph.info.part.{{forecast.mpas.execution.batchargs.cores }}": '{{ user.experiment_dir }}/{{ user.mesh_label }}.graph.info.part.{{forecast.mpas.execution.batchargs.cores }}' - stream_list.atmosphere.diagnostics: "{{ user.mpas_app }}/parm/mpas/stream_list.atmosphere.diagnostics" - stream_list.atmosphere.output: "{{ user.mpas_app }}/parm/mpas/stream_list.atmosphere.output" - stream_list.atmosphere.surface: "{{ user.mpas_app }}/parm/mpas/stream_list.atmosphere.surface" files_to_link: <<: *mpas_init_files_to_link - CCN_ACTIVATE.BIN: "{{ platform.hrrr_fix }}/CCN_ACTIVATE.BIN" + ccn_activate.bin: "{{ platform.hrrr_fix }}/physics/hrrrv5/ccn_activate.bin" + freeze_water_data_tempo_v3: "{{ user.mpas_app }}/src/MPAS-Model/freeze_water_data_tempo_v3" + qr_acr_qs_data_tempo_v3: "{{ user.mpas_app }}/src/MPAS-Model/qr_acr_qs_data_tempo_v3" + qr_acr_qg_data_tempo_v3: "{{ user.mpas_app }}/src/MPAS-Model/qr_acr_qg_data_tempo_v3" CCN_ACTIVATE_DATA: "{{ user.mpas_app }}/src/MPAS-Model/CCN_ACTIVATE_DATA" COMPATIBILITY: "{{ user.mpas_app }}/src/MPAS-Model/src/core_atmosphere/physics/physics_wrf/files/COMPATIBILITY" - MP_TEMPO_QRacrQS_DATA.DBL: "{{ user.mpas_app }}/src/MPAS-Model/MP_TEMPO_QRacrQS_DATA.DBL" - MP_TEMPO_freezeH2O_DATA.DBL: "{{ user.mpas_app }}/src/MPAS-Model/MP_TEMPO_freezeH2O_DATA.DBL" - MP_TEMPO_QIautQS_DATA.DBL: "{{ user.mpas_app }}/src/MPAS-Model/MP_TEMPO_QIautQS_DATA.DBL" - MP_TEMPO_HAILAWARE_QRacrQG_DATA.DBL: "{{ user.mpas_app }}/src/MPAS-Model/MP_TEMPO_HAILAWARE_QRacrQG_DATA.DBL" lateral_boundary_conditions: interval_hours: !int "{{ user.lbcs.interval_hours }}" offset: 0 @@ -278,13 +280,14 @@ forecast: config_radtsw_interval: '00:15:00' config_bucket_update: 'none' config_physics_suite: 'hrrrv5' - config_tempo_aerosolaware: true - config_tempo_hailaware: true config_gwdo_scheme: 'bl_ugwp_gwdo' config_lsm_scheme: 'sf_ruc' num_soil_layers: 9 config_pbl_scheme: 'bl_mynnedmf' - + physics_mp_tempo: + config_tempo_aerosolaware: true + config_tempo_hailaware: true + config_tempo_ml_for_bl_nc: true rundir: '{{ user.experiment_dir }}/{{ cycle.strftime("%Y%m%d%H") }}/forecast' streams: input: @@ -349,8 +352,8 @@ post: execution: batchargs: walltime: 00:30:00 - cores: 768 - nodes: 24 + tasks_per_node: 160 + nodes: 1 native: - --exclusive fixdir: /path/to/fix/files diff --git a/ush/experiment_gen.py b/ush/experiment_gen.py index f18c14a5..c6e41092 100755 --- a/ush/experiment_gen.py +++ b/ush/experiment_gen.py @@ -79,7 +79,7 @@ def main(): validated = validate(experiment_config.as_dict()) experiment_dir, experiment_file = setup_experiment_directory(validated) generate_workflow_files(experiment_config, experiment_file, mpas_app, user_config, validated) - stage_grid_files(experiment_config, experiment_dir, validated) + stage_grid_files(experiment_config, experiment_dir) def parse_args() -> list[Path]: @@ -153,19 +153,17 @@ def setup_experiment_directory(validated: Config) -> tuple[Path, Path]: return experiment_dir, experiment_file -def stage_grid_files( - experiment_config: YAMLConfig, experiment_dir: Path, validated: Config -) -> None: +def stage_grid_files(experiment_config: YAMLConfig, experiment_dir: Path) -> None: """ Create grid files for each required processor count, if they don't already exist. """ - mesh_file_name = f"{validated.user.mesh_label}.graph.info" - mesh_file_path = Path(experiment_config["data"]["mesh_files"]) / mesh_file_name + graphinfo_filename = experiment_config["data"]["graphinfo_filename"] + graphinfo_filepath = Path(experiment_config["data"]["mesh_files"]) / graphinfo_filename for nprocs in required_nprocs(experiment_config): - part_file = experiment_dir / f"{mesh_file_path.name}.part.{nprocs}" + part_file = experiment_dir / f"{graphinfo_filepath.name}.part.{nprocs}" if not part_file.is_file(): logging.info("Creating grid file for %s procs", nprocs) - create_grid_files(experiment_dir, mesh_file_path, nprocs) + create_grid_files(experiment_dir, graphinfo_filepath, nprocs) def validate_driver_blocks(validated_blocks: list[str], workflow_config: YAMLConfig) -> None: diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 98df0c10..4f9bc45d 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -478,7 +478,7 @@ def try_data_store( for fs_copy_config in fs_copy_configs: getter = fs.link if symlink else fs.copy files_copied = getter(config=fs_copy_config, target_dir=outpath, cycle=cycle) - if files_copied["ready"] and not files_copied["not-ready"]: + if files_copied["ready"] and not files_copied["notready"]: logging.info(files_copied) return True, fs_copy_config