From fe2fba9441c7395eef87ce3c1da5bde92a9a4cdd Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 20 Aug 2024 07:59:38 -0400 Subject: [PATCH 1/6] Update to match GEOSgcm main as of 2024-Aug-20 --- .circleci/config.yml | 109 +--- .editorconfig | 27 + .github/workflows/enforce-labels.yml | 10 +- .github/workflows/workflow.yml | 28 +- .gitignore | 11 + CMakeLists.txt | 69 +- CMakePresets.json | 49 +- README.md | 87 ++- components.yaml | 31 +- parallel_build.csh | 16 +- src/Applications/GEOSctm_App/CMakeLists.txt | 4 +- src/Applications/GEOSctm_App/ctm_setup | 662 +++++++------------- 12 files changed, 521 insertions(+), 582 deletions(-) create mode 100644 .editorconfig diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f118e8..5782f05 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,85 +1,42 @@ version: 2.1 -executors: - gfortran-large: - docker: - - image: gmao/ubuntu20-geos-env-mkl:v6.2.8-openmpi_4.0.6-gcc_11.2.0 - auth: - username: $DOCKERHUB_USER - password: $DOCKERHUB_AUTH_TOKEN - environment: - OMPI_ALLOW_RUN_AS_ROOT: 1 - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 - OMPI_MCA_btl_vader_single_copy_mechanism: none - resource_class: large - #MEDIUM# resource_class: medium +parameters: + GHA_Actor: + type: string + default: "" + GHA_Action: + type: string + default: "" + GHA_Event: + type: string + default: "" + GHA_Meta: + type: string + default: "" - ifort-large: - docker: - - image: gmao/ubuntu20-geos-env:v6.2.8-intelmpi_2021.2.0-intel_2021.2.0 - auth: - username: $DOCKERHUB_USER - password: $DOCKERHUB_AUTH_TOKEN - resource_class: large - #MEDIUM# resource_class: medium +# Anchors to prevent forgetting to update a version +os_version: &os_version ubuntu20 +baselibs_version: &baselibs_version v7.25.0 +bcs_version: &bcs_version v11.5.0 +tag_build_arg_name: &tag_build_arg_name fv3version + +orbs: + ci: geos-esm/circleci-tools@2 workflows: build-test: jobs: - - build-GEOSctm: - name: build-GEOSctm-on-<< matrix.compiler >> + + # Build GEOSfvdycore + - ci/build: + name: build-GEOSfvdycore-as-<< matrix.fv_precision >>-on-<< matrix.compiler >> + context: + - docker-hub-creds matrix: parameters: - compiler: [gfortran, ifort] - context: - - docker-hub-creds - -jobs: - build-GEOSctm: - parameters: - compiler: - type: string - executor: << parameters.compiler >>-large - working_directory: /root/project - steps: - - checkout: - path: GEOSctm - - run: - name: "Versions etc" - command: mpirun --version && << parameters.compiler>> --version && echo $BASEDIR && pwd && ls - - run: - name: "Mepo clone external repos" - command: | - cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm - mepo clone - mepo status - - run: - name: "Mepo checkout-if-exists" - command: | - cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm - echo "${CIRCLE_BRANCH}" - if [ "${CIRCLE_BRANCH}" != "develop" ] && [ "${CIRCLE_BRANCH}" != "main" ] - then - mepo checkout-if-exists ${CIRCLE_BRANCH} - fi - mepo status - - run: - name: "CMake" - command: | - mkdir -p /logfiles - cd ${CIRCLE_WORKING_DIRECTORY}/GEOSctm - mkdir -p ${CIRCLE_WORKING_DIRECTORY}/workspace/build-GEOSctm - cd ${CIRCLE_WORKING_DIRECTORY}/workspace/build-GEOSctm - cmake ${CIRCLE_WORKING_DIRECTORY}/GEOSctm -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=<< parameters.compiler >> -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=${CIRCLE_WORKING_DIRECTORY}/workspace/install-GEOSctm -DUSE_F2PY=OFF |& tee /logfiles/cmake.log - - run: - name: "Build and install" - command: | - cd ${CIRCLE_WORKING_DIRECTORY}/workspace/build-GEOSctm - make -j"$(nproc)" install |& tee /logfiles/make.log - #MEDIUM# make -j4 install |& tee /logfiles/make.log - - run: - name: "Compress artifacts" - command: | - gzip -9 /logfiles/* - - store_artifacts: - path: /logfiles + compiler: [ifort, gfortran] + fv_precision: ["R8"] + baselibs_version: *baselibs_version + repo: GEOSctm + mepodevelop: false + persist_workspace: false # Needs to be true to run fv3/gcm experiment, costs extra \ No newline at end of file diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..986cead --- /dev/null +++ b/.editorconfig @@ -0,0 +1,27 @@ +# Global Editor Config for MAPL +# +# This is an ini style configuration. See http://editorconfig.org/ for more information on this file. +# +# Top level editor config. +root = true + +# Always use Unix style new lines with new line ending on every file and trim whitespace +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +# Python: PEP8 defines 4 spaces for indentation +[*.py] +indent_style = space +indent_size = 4 + +# YAML format, 2 spaces +[{*.yaml,*.yml}] +indent_style = space +indent_size = 2 + +# CMake (from KitWare: https://github.com/Kitware/CMake/blob/master/.editorconfig) +[{CMakeLists.txt,*.cmake,*.rst}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/enforce-labels.yml b/.github/workflows/enforce-labels.yml index 6e1720e..4a9320c 100644 --- a/.github/workflows/enforce-labels.yml +++ b/.github/workflows/enforce-labels.yml @@ -8,18 +8,20 @@ jobs: require-label: runs-on: ubuntu-latest steps: - - uses: mheap/github-action-required-labels@v2 + - uses: mheap/github-action-required-labels@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: mode: minimum count: 1 - labels: "0 diff,0 diff trivial,Non 0-diff,0 diff structural,0-diff trivial,Not 0-diff,0-diff,automatic,0-diff uncoupled" + labels: "0 diff,0 diff trivial,Non 0-diff,0 diff structural,0-diff trivial,Not 0-diff,0-diff,automatic,0-diff uncoupled,github_actions" add_comment: true + message: "This PR is being prevented from merging because you have not added one of our required labels: {{ provided }}. Please add one so that the PR can be merged." + blocking-label: runs-on: ubuntu-latest steps: - - uses: mheap/github-action-required-labels@v2 + - uses: mheap/github-action-required-labels@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -27,3 +29,5 @@ jobs: count: 0 labels: "Contingent - DNA,Needs Lead Approval,Contingent -- Do Not Approve" add_comment: true + message: "This PR is being prevented from merging because you have added one of our blocking labels: {{ provided }}. You'll need to remove it before this PR can be merged." + diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 0671acb..9d6c667 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -6,6 +6,8 @@ on: # Do not run if the only files changed cannot affect the build paths-ignore: - "**.md" + - "**.json" + - "ChangeLog-PreJason.txt" - "parallel_build.csh" - ".github/CODEOWNERS" - ".codebuild/**" @@ -17,33 +19,41 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')" runs-on: ubuntu-latest container: - image: gmao/ubuntu20-geos-env-mkl:v6.2.4-openmpi_4.0.5-gcc_10.3.0 - credentials: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + image: gmao/ubuntu20-geos-env:v7.25.0-intelmpi_2021.13-intel_2024.2 + # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495 + # It seems like we might not need secrets on GitHub Actions which is good for forked + # pull requests + #credentials: + #username: ${{ secrets.DOCKERHUB_USERNAME }} + #password: ${{ secrets.DOCKERHUB_TOKEN }} env: OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 OMPI_MCA_btl_vader_single_copy_mechanism: none steps: + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.5.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} - name: Checkout CTM - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Set all directories as git safe + run: | + git config --global --add safe.directory '*' - name: Versions etc. run: | - gfortran --version + ifort --version mpirun --version echo $BASEDIR - name: Mepo clone external repos run: | mepo clone mepo status - mepo status - name: Update other branches if: "!contains('refs/heads/main,refs/heads/develop', github.ref)" @@ -54,7 +64,7 @@ jobs: run: | mkdir build cd build - cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' -DUSE_F2PY=OFF + cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=ifort -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS='--oversubscribe' -DUSE_F2PY=OFF - name: Build run: | cd build diff --git a/.gitignore b/.gitignore index 8fd879f..f3adeaf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,14 @@ parallel_build.o* log.* CMakeUserPresets.json + +# Ignore possible symlinked directories +build-* +install-* + +*.swp +*.swo +.DS_Store +*# +.#* +**/CVS/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fc8ecc..ff16a45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,10 @@ -cmake_minimum_required (VERSION 3.13) +cmake_minimum_required (VERSION 3.17) cmake_policy (SET CMP0053 NEW) cmake_policy (SET CMP0054 NEW) project ( GEOSctm - VERSION 2.0.1 + VERSION 2.3.0 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF if ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}") @@ -34,10 +34,64 @@ set (FV_PRECISION "R8" CACHE STRING "Precision of FV3 core (R4, R4R8, R8)") foreach (dir cmake @cmake cmake@) if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/${dir}) list (APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/${dir}") + set (ESMA_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}/${dir}" CACHE PATH "Path to ESMA_cmake code") endif () endforeach () + +# We need to find MPI before we go into esma +# for the MPI stack detection to work +set(MPI_DETERMINE_LIBRARY_VERSION TRUE) +find_package(MPI) + include (esma) +# Add CMake for when not using Baselibs +if (NOT Baselibs_FOUND) + + find_package(NetCDF REQUIRED C Fortran) + add_definitions(-DHAS_NETCDF4) + add_definitions(-DHAS_NETCDF3) + add_definitions(-DNETCDF_NEED_NF_MPIIO) + add_definitions(-DHAS_NETCDF3) + + find_package(HDF5 REQUIRED) + if(HDF5_IS_PARALLEL) + add_definitions(-DH5_HAVE_PARALLEL) + endif() + + if (NOT TARGET ESMF::ESMF) + find_package(ESMF 8.6.1 MODULE REQUIRED) + target_link_libraries(ESMF::ESMF INTERFACE MPI::MPI_Fortran) + + # GEOS uses lowercase target due to historical reasons but + # the latest FindESMF.cmake file from ESMF produces an ESMF::ESMF target. + if (NOT TARGET esmf) + add_library(esmf ALIAS ESMF::ESMF) + endif () + if (NOT TARGET ESMF) + add_library(ESMF ALIAS ESMF::ESMF) + endif () + endif () + + find_package(GFTL_SHARED REQUIRED) + + find_package(ZLIB REQUIRED) + # Another issue with historical reasons, old/wrong zlib target used in GEOS + add_library(ZLIB::zlib ALIAS ZLIB::ZLIB) + + # Using FMS from spack requires updates to fvdycore due to interface changes + # in FMS 2022. This is commented for now until this transition can occur. + ################################################# + # find_package(FMS QUIET COMPONENTS R4 R8) # + # if (FMS_FOUND) # + # # We need aliases due to historical reasons # + # add_library(fms_r4 ALIAS FMS::fms_r4) # + # add_library(fms_r8 ALIAS FMS::fms_r8) # + # endif () # + ################################################# + +endif () + ecbuild_declare_project() # Generic DFLAGS @@ -63,5 +117,14 @@ install( DESTINATION ${CMAKE_INSTALL_PREFIX} ) -# Adds abiilty to tar source +# Adds ability to tar source include (esma_cpack) + +# This installs a tarball of the source code +# in the installation directory. +# MUST BE THE LAST CODE IN THIS FILE +option(INSTALL_SOURCE_TARFILE "Create and install source tarfile" OFF) +if(INSTALL_SOURCE_TARFILE) + install(CODE "set(CMAKE_PROJECT_NAME \"${CMAKE_PROJECT_NAME}\")") + install(SCRIPT "${ESMA_CMAKE_PATH}/esma_postinstall.cmake") +endif() diff --git a/CMakePresets.json b/CMakePresets.json index b616a27..60c99f0 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,23 +1,14 @@ { - "version": 3, + "version": 7, "cmakeMinimumRequired": { "major": 3, - "minor": 21, + "minor": 27, "patch": 0 }, + "include": [ + "presets/CMake$penv{CMAKE_PRESET_NAME}Presets.json" + ], "configurePresets": [ - { - "name": "base-configure", - "hidden": true, - "displayName": "Base Configure Settings", - "description": "Sets build and install directories", - "binaryDir": "${sourceDir}/build-${presetName}", - "cacheVariables": { - "BASEDIR": "$env{BASEDIR}", - "CMAKE_INSTALL_PREFIX": "${sourceDir}/install-${presetName}", - "CMAKE_BUILD_TYPE": "${presetName}" - } - }, { "name": "base-gnu", "hidden": true, @@ -38,37 +29,55 @@ "name": "Release", "inherits": "base-gnu", "displayName": "Release Configure", - "description": "Release build using GNU Make generator" + "description": "Release build using GNU Make generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } }, { "name": "Debug", "inherits": "base-gnu", "displayName": "Debug Configure", - "description": "Debug build using GNU Make generator" + "description": "Debug build using GNU Make generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } }, { "name": "Aggressive", "inherits": "base-gnu", "displayName": "Aggressive Configure", - "description": "Aggressive build using GNU Make generator" + "description": "Aggressive build using GNU Make generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Aggressive" + } }, { "name": "Release-Ninja", "inherits": "base-ninja", "displayName": "Release Ninja Configure", - "description": "Release build using Ninja generator" + "description": "Release build using Ninja generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } }, { "name": "Debug-Ninja", "inherits": "base-ninja", "displayName": "Debug Ninja Configure", - "description": "Debug build using Ninja generator" + "description": "Debug build using Ninja generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } }, { "name": "Aggressive-Ninja", "inherits": "base-ninja", "displayName": "Aggressive Ninja Configure", - "description": "Aggressive build using Ninja generator" + "description": "Aggressive build using Ninja generator", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Aggressive" + } } ], "buildPresets": [ diff --git a/README.md b/README.md index c889162..c6e8bdb 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,21 @@ In your `.bashrc` or `.tcshrc` or other rc file add a line: ##### NCCS + +NCCS currently has two different OSs. So you'll need to use different modulefiles depending on which OS you are using. + +###### SLES 12 + ``` module use -a /discover/swdev/gmao_SIteam/modulefiles-SLES12 ``` +###### SLES 15 + +``` +module use -a /discover/swdev/gmao_SIteam/modulefiles-SLES15 +``` + ##### NAS ``` module use -a /nobackup/gmao_SIteam/modulefiles @@ -50,8 +61,9 @@ is enabled on GitHub. To clone the GEOSctm using the SSH url (starts with `git@github.com`), you run: ``` -git clone git@github.com:GEOS-ESM/GEOSctm.git +git clone -b vX.Y.Z git@github.com:GEOS-ESM/GEOSctm.git ``` +where `vX.Y.Z` is a tag from a [GEOSctm release](https://github.com/GEOS-ESM/GEOSctm/releases). Note if you don't use `-b`, you will get the `main` branch and that can change from day-to-day. ###### Permission denied (publickey) @@ -76,8 +88,9 @@ use the SSH URL through. To clone the model through HTTPS you run: ``` -git clone https://github.com/GEOS-ESM/GEOSctm.git +git clone -b vX.Y.Z https://github.com/GEOS-ESM/GEOSctm.git ``` +where `vX.Y.Z` is a tag from a [GEOSctm release](https://github.com/GEOS-ESM/GEOSctm/releases). Note if you don't use `-b`, you will get the `main` branch and that can change from day-to-day. Note that if you use the HTTPS URL and have 2FA set up on GitHub, you will need to use [personal access @@ -88,8 +101,9 @@ as a password. You can also use the [GitHub CLI](https://cli.github.com/) with: ``` -gh repo clone GEOS-ESM/GEOSctm +gh repo clone GEOS-ESM/GEOSctm -- -b vX.Y.Z ``` +where `vX.Y.Z` is a tag from a [GEOSctm release](https://github.com/GEOS-ESM/GEOSctm/releases). Note if you don't use `-b`, you will get the `main` branch and that can change from day-to-day. Note that when you first use `gh`, it will ask what your preferred git protocol is (https or ssh) to use "underneath". The caveats above will apply to whichever @@ -101,10 +115,53 @@ you choose. If all you wish is to build the model, you can run `parallel_build.csh` from a head node. Doing so will checkout all the external repositories of the model and build it. When done, the resulting model build will be found in `build/` and the installation will be found in `install/` with setup scripts like `ctm_setup` in `install/bin`. -#### Debug Version of GEOS +#### Building at NCCS (Multiple OSs) + +In all the examples below, NCCS builds will act differently. Because NCCS currently has two different OSs, when you use +`parallel_build.csh` you will see that the `build` and `install` directories will be appended with `-SLES12` or `-SLES15` depending +on where you submitted to. When NCCS moves to a single OS again, this will be removed. + +Note that if you use the `-builddir` and `-installdir` options, you can override this behavior and no OS will be automatically +appended. + +#### Develop Version of GEOS CTM + +`parallel_build.csh` provides a special flag for checking out the +development branches of GMAO_Shared and GEOS_Util. If you run: + +``` +parallel_build.csh -develop +``` +then `mepo` will run: + +``` +mepo develop GMAO_Shared GEOS_Util +``` + +#### Debug Version of GEOS CTM To obtain a debug version, you can run `parallel_build.csh -debug` which will build with debugging flags. This will build in `build-Debug/` and install into `install-Debug/`. +#### Do not create and install source tarfile with parallel_build + +Note that running with `parallel_build.csh` will create and install a tarfile of the source code at build time. If you wish to avoid +this, run `parallel_build.csh` with the `-no-tar` option. + +#### Passing additional CMake options to `parallel_build.csh` + +While `parallel_build.csh` has many options, it does not cover all possible CMake options possible in GEOSctm. If you wish to +pass additional CMake options to `parallel_build.csh`, you can do so by using `--` and then the CMake options. Note that *anything* +after the `--` will be interpreted as a CMake option, which could lead to build issues if not careful. + +For example, if you want to build a develop Debug build on Cascade Lake while turning on StratChem reduced mechanism and the CODATA +2018 options: + +``` +parallel_build.csh -develop -debug -cas -- -DSTRATCHEM_REDUCED_MECHANISM=ON -DUSE_CODATA_2018_CONSTANTS=ON +``` + +As noted above all the "regular" `parallel_build.csh` options must be listed before the `--` flag. + --- ### Multiple Steps for Building the Model @@ -126,6 +183,20 @@ The first command initializes the multi-repository and the second one clones and assembles all the sub-repositories according to `components.yaml` +#### Checking out develop branches of GMAO_Shared and GEOS_Util + +To get development branches of GMAO_Shared and GEOS_Util (a la +the `-develop` flag for `parallel_build.csh`, one needs to run the +equivalent `mepo` command. As mepo itself knows (via `components.yaml`) what the development branch of each +subrepository is, the equivalent of `-develop` for `mepo` is to +checkout the development branches of GMAO_Shared and GEOS_Util: +``` +mepo develop GMAO_Shared GEOS_Util +``` + +This must be done *after* `mepo clone` as it is running a git command in +each sub-repository. + #### Build the Model ##### Load Compiler, MPI Stack, and Baselibs @@ -157,6 +228,14 @@ This will install to a directory parallel to your `build` directory. If you pref ``` and CMake will install there. +###### Create and install source tarfile + +Note that running with `parallel_build.csh` will create and install a tarfile of the source code at build time. But if CMake is run by hand, this is not the default action (as many who build with CMake by hand are developers and not often running experiments). In order to enable this at install time, add: +``` +-DINSTALL_SOURCE_TARFILE=ON +``` +to your CMake command. + ##### Build and Install with Make ``` make -jN install diff --git a/components.yaml b/components.yaml index e48d915..c830dfc 100644 --- a/components.yaml +++ b/components.yaml @@ -5,62 +5,69 @@ GEOSctm: env: local: ./@env remote: ../ESMA_env.git - tag: v3.4.0 + tag: v4.29.0 develop: main cmake: local: ./@cmake remote: ../ESMA_cmake.git - tag: v3.6.2 + tag: v3.48.0 develop: develop ecbuild: local: ./@cmake/@ecbuild remote: ../ecbuild.git - tag: geos/v1.0.6 + tag: geos/v1.3.0 + +NCEP_Shared: + local: ./src/Shared/@NCEP_Shared + remote: ../NCEP_Shared.git + tag: v1.3.0 + sparse: ./config/NCEP_Shared.sparse + develop: main GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - tag: v1.4.10 + tag: v1.9.8 sparse: ./config/GMAO_Shared.sparse develop: main MAPL: local: ./src/Shared/@MAPL remote: ../MAPL.git - tag: v2.8.6 + tag: v2.47.1 develop: develop FMS: local: ./src/Shared/@FMS remote: ../FMS.git - tag: geos/2019.01.02+noaff.7 + tag: geos/2019.01.02+noaff.10 develop: geos/release/2019.01 FVdycoreCubed_GridComp: local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp remote: ../FVdycoreCubed_GridComp.git - tag: v1.2.17 + tag: v2.11.1 develop: develop fvdycore: local: ./src/Components/GEOSctm_GridComp/@FVdycoreCubed_GridComp/@fvdycore remote: ../GFDL_atmos_cubed_sphere.git - tag: geos/v1.1.7 + tag: geos/v2.9.0 develop: geos/develop GEOSchem_GridComp: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp remote: ../GEOSchem_GridComp.git - tag: v1.6.0 + tag: v1.14.0 develop: develop HEMCO: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/Shared/HEMCO/@HEMCO remote: ../HEMCO.git - tag: geos/v2.2.1 - develop: geos/develop + tag: geos/v2.2.3 + develop: geos/develop geos-chem: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/GEOSCHEMchem_GridComp/@geos-chem @@ -72,5 +79,5 @@ GOCART: local: ./src/Components/GEOSctm_GridComp/@GEOSchem_GridComp/@GOCART remote: ../GOCART.git sparse: ./config/GOCART.sparse - tag: v1.0.1 + tag: sdr_v2.2.1.2 develop: develop diff --git a/parallel_build.csh b/parallel_build.csh index 8bde58a..2ffb12d 100755 --- a/parallel_build.csh +++ b/parallel_build.csh @@ -32,8 +32,8 @@ end if (-d ${ESMADIR}/@env || -d ${ESMADIR}/env@ || -d ${ESMADIR}/env) then if ( "$DEVELOP" == "TRUE" ) then - echo "Checking out development branches of GMAO_Shared" - mepo develop GMAO_Shared + echo "Checking out development branches of GMAO_Shared and GEOS_Util" + mepo develop GMAO_Shared GEOS_Util endif else if ($?PBS_JOBID || $?SLURM_JOBID) then @@ -42,14 +42,14 @@ else echo " Please run from a head node" exit 1 else - echo "Running mepo initialization" - mepo init - mepo clone + echo "Running mepo initialization" + mepo init + mepo clone if ( "$DEVELOP" == "TRUE" ) then - echo "Checking out development branches of GEOSgcm_GridComp, GEOSgcm_App, and GMAO_Shared" - mepo develop GMAO_Shared - endif + echo "Checking out development branches of GMAO_Shared and GEOS_Util" + mepo develop GMAO_Shared GEOS_Util endif + endif endif # Now reset argv diff --git a/src/Applications/GEOSctm_App/CMakeLists.txt b/src/Applications/GEOSctm_App/CMakeLists.txt index e5fc6a2..b86c49f 100644 --- a/src/Applications/GEOSctm_App/CMakeLists.txt +++ b/src/Applications/GEOSctm_App/CMakeLists.txt @@ -2,7 +2,7 @@ ecbuild_add_executable ( TARGET GEOSctm.x SOURCES GEOSctm.F90 LIBS GEOSctm_GridComp - ) + ) set (CMAKE_Fortran_FLAGS_RELEASE "${GEOS_Fortran_FLAGS_VECT}") @@ -35,7 +35,7 @@ install ( FILES fvcore_layout.rc logging.yaml DESTINATION etc ) - + if(HYDROSTATIC) set(CFG_HYDROSTATIC TRUE) else() diff --git a/src/Applications/GEOSctm_App/ctm_setup b/src/Applications/GEOSctm_App/ctm_setup index ce6250b..195cb0f 100755 --- a/src/Applications/GEOSctm_App/ctm_setup +++ b/src/Applications/GEOSctm_App/ctm_setup @@ -17,7 +17,7 @@ ####################################################################### # Define Colors -# Note: For No Colors, set C1 and C2 to NONE +# Note: For No Colors, set C1 and C2 to NONE ####################################################################### set BLACK = `tput setaf 0` @@ -72,7 +72,6 @@ endif ####################################################################### # Set default behavior of switches -set NOCVS = TRUE set GPU = FALSE set LINKX = FALSE @@ -93,11 +92,6 @@ while ( $#argv > 0 ) set GPU = TRUE breaksw - # Do not archive the source - case --nocvs: - set NOCVS = TRUE - breaksw - # Symlink GEOSctm.x case --link: set LINKX = TRUE @@ -112,46 +106,13 @@ while ( $#argv > 0 ) end ####################################################################### -# CVS: Use CVS functionality only at NCCS or NAS +# Determine site ####################################################################### setenv NODE `uname -n` setenv ARCH `uname -s` setenv SITE `awk '{print $2}' $ETCDIR/SITE.rc` -if ( $SITE != 'NCCS' && $SITE != 'NAS' ) then - set NOCVS = TRUE -endif - -####################################################################### -# CVS: Test for Environment Variable -####################################################################### - -if ( $?ESMA_NOCVS ) then - set NOCVS = TRUE -endif - -####################################################################### -# CVS: Test if we are on a compute node -####################################################################### - -if ( $NOCVS != "TRUE" ) then - -if ( $SITE == 'NCCS' ) then - if ( ($NODE =~ borg*) || ($NODE =~ warp*) ) then - goto ONCOMPUTE - endif - -else if ( $SITE == 'NAS' ) then - if ( ($NODE =~ r[0-9]*i[0-9]*n[0-9]*) || \ - ($NODE =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*) || \ - ($NODE =~ maia*) ) then - goto ONCOMPUTE - endif -endif - -endif - ####################################################################### # Test for Compiler and MPI Setup ####################################################################### @@ -172,7 +133,7 @@ else if ( `echo $BASEDIR | grep -i mpt` != '') then set MPI = mpt else # Assume default is Intel MPI in case of older baselibs - set MPI = intelmpi + set MPI = intelmpi endif ####################################################################### @@ -194,7 +155,6 @@ end CTMTAG: set CTMTAG = `cat ${ETCDIR}/.GEOSCTM_VERSION` -#echo "${C1}CVS BASE Source Tag${CN} used for Experiment: ${C2}${CTMTAG}${CN}" ####################################################################### # Test to see if you want to CLONE old experiment @@ -228,7 +188,7 @@ else endif ####################################################################### -# Continue to enter in experiment parameters +# Continue to enter in experiment parameters ####################################################################### HRCODE: @@ -373,33 +333,50 @@ endif ASKPROC: if ( $SITE == 'NCCS' ) then - echo "Enter the ${C1}Processor Type${CN} you wish to run on:" - echo " ${C2}hasw (Haswell)${CN}" - echo " ${C2}sky (Skylake)${CN} (default)" - echo " ${C2}cas (Cascade Lake)${CN}" - echo " " - set MODEL = `echo $<` - set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` - if ( .$MODEL == .) then - set MODEL = 'sky' - endif - if( $MODEL != 'hasw' & \ - $MODEL != 'sky' & \ - $MODEL != 'cas' ) goto ASKPROC + set BUILT_ON_SLES15 = @BUILT_ON_SLES15@ - if ( $MODEL == 'hasw') then - set NCPUS_PER_NODE = 28 - else if ($MODEL == 'sky') then - set NCPUS_PER_NODE = 40 - else if ($MODEL == 'cas') then - # NCCS currently recommends that users do not run with - # 48 cores per node on SCU16 due to OS issues and - # recommends that CPU-intensive works run with 46 or less - # cores. As 45 is a multiple of 3, it's the best value - # that doesn't waste too much - #set NCPUS_PER_NODE = 48 - set NCPUS_PER_NODE = 45 + if ("$BUILT_ON_SLES15" == "TRUE") then + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}mil (Milan)${CN} (default)" + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'mil' + endif + + if( $MODEL != 'mil' ) goto ASKPROC + + if ($MODEL == 'mil') then + # We save a couple processes for the kernel + set NCPUS_PER_NODE = 126 + endif + else + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}sky (Skylake)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'sky' + endif + + if( $MODEL != 'sky' & \ + $MODEL != 'cas' ) goto ASKPROC + + if ($MODEL == 'sky') then + set NCPUS_PER_NODE = 40 + else if ($MODEL == 'cas') then + # NCCS currently recommends that users do not run with + # 48 cores per node on SCU16 due to OS issues and + # recommends that CPU-intensive works run with 46 or less + # cores. As 45 is a multiple of 3, it's the best value + # that doesn't waste too much + #set NCPUS_PER_NODE = 48 + set NCPUS_PER_NODE = 45 + endif endif else if ( $SITE == 'NAS' ) then @@ -638,7 +615,7 @@ if( $GEOSCTM_IM == "c360" ) then set CHEM_DT = $DT set GEOSCTM_IM = 360 set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` - set NX = 12 + set NX = 12 set NY = `expr $NX \* 6` set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` @@ -706,7 +683,7 @@ if( $GEOSCTM_IM == "c1440" ) then set CHEM_DT = $DT set GEOSCTM_IM = 1440 set GEOSCTM_JM = `expr $GEOSCTM_IM \* 6` - set NX = 30 + set NX = 30 set NY = `expr $NX \* 6` set HYDROSTATIC = $USE_HYDROSTATIC set HIST_IM = `expr $GEOSCTM_IM \* 4` @@ -907,7 +884,7 @@ set HH = `expr $DT \/ 3600` set MM = `expr \( $DT \- $HH \* 3600 \) \/ 60` set SS = `expr $DT \- $HH \* 3600 \- $MM \* 60` set nhmsDT = `expr $HH \* 10000 \+ $MM \* 100 \+ $SS` - + ####################################################################### # Check for AEROSOL/Chemistry Models ####################################################################### @@ -1070,9 +1047,9 @@ if( $ctmTRACER == TRUE ) then endif if( $ctmIPT == TRUE ) then - set AERO_PROVIDER = - set RATS_PROVIDER = - set OANA_PROVIDER = + set AERO_PROVIDER = + set RATS_PROVIDER = + set OANA_PROVIDER = endif set PCHEM = "#" @@ -1148,13 +1125,8 @@ if( $SITE == 'NAS' ) then setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "8:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "8:00:00" # Wallclock Time for ctm_archive.j - if( $GPU == "TRUE" ) then - set NCPUS_PER_NODE = 16 # CPUS per node - set MODEL = "san_gpu" # Model of CPU - set QTYPE = "gpu_k40" # Queue to use - else - set QTYPE = "normal" # Queue to use - endif + + set QTYPE = "normal" # Queue to use #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` @@ -1178,8 +1150,8 @@ if( $SITE == 'NAS' ) then setenv CHMDIR /nobackup/gmao_SIteam/ModelData/fvInput_nc3 # locations of Aerosol Chemistry BCs setenv WRKDIR /nobackup/$LOGNAME # user work directory setenv COUPLEDIR /nobackup/gmao_SIteam/ModelData/coupled/Forcings # Coupled Ocean/Atmos Forcing - setenv CPEXEC 'mcp -a' # Copy utility for large copies - setenv TAREXEC mtar # Tar utility for large archives + setenv CPEXEC 'cp -a' # Copy utility for large copies + setenv TAREXEC tar # Tar utility for large archives else if( $SITE == 'NCCS' ) then setenv BATCH_CMD "sbatch" # SLURM Batch command setenv BATCH_GROUP "SBATCH --account=" # SLURM Syntax for account name @@ -1192,10 +1164,6 @@ else if( $SITE == 'NCCS' ) then setenv POST_T "8:00:00" # Wallclock Time for ctm_post.j setenv PLOT_T "12:00:00" # Wallclock Time for ctm_plot.j setenv ARCHIVE_T "2:00:00" # Wallclock Time for ctm_archive.j - if( $GPU == "TRUE" ) then - set NCPUS_PER_NODE = 16 # CPUS per node - set MODEL = "k40" # Model of CPU - endif #@ NODES = `echo "($MODEL_NPES + $NCPUS_PER_NODE - 1)/$NCPUS_PER_NODE" | bc` @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` @@ -1396,7 +1364,7 @@ while( $check == FALSE ) else setenv HOMDIR $HOMDIR_def endif - + if( "$EXPID" != `basename $HOMDIR` ) then echo "\!\! The ${C1}HOME${CN} Directory MUST point to the ${C1}EXPID${CN}: ${C2}${EXPID}${CN}" echo " " @@ -1429,7 +1397,7 @@ while( $check == FALSE ) else setenv EXPDIR $EXPDIR_def endif - + if( "$EXPID" != `basename $EXPDIR` ) then echo "\!\! The ${C1}EXPERIMENT${CN} Directory MUST point to the ${C1}EXPID${CN}: ${C2}${EXPID}${CN}" echo " " @@ -1484,7 +1452,7 @@ echo $EXPDIRroot > $HOME/.EXPDIRroot setenv CNVDIR $EXPDIR/convert ####################################################################### -# Locate Build Directory +# Locate Build Directory ####################################################################### # GEOSctm Build Directory @@ -1595,79 +1563,89 @@ set RESTART_BY_OSERVER = NO /bin/rm -f $HOMDIR/SETENV.commands + if( $MPI == openmpi ) then # Open MPI and GEOS has issues with restart writing. Having the # oserver write them can be orders of magnitude faster + set RESTART_BY_OSERVER = YES -# This turns off an annoying warning when running -# Open MPI on a system where TMPDIRs are on a networked -# file system +# Testing by Bill Putman determined some useful +# Open MPI parameters. Testing shows these work +# on both OSs at NCCS and on macOS cat > $HOMDIR/SETENV.commands << EOF - setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 +# Turn off warning about TMPDIR on NFS +setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 +# pre-connect MPI procs on mpi_init +setenv OMPI_MCA_mpi_preconnect_all 1 +setenv OMPI_MCA_coll_tuned_bcast_algorithm 7 +setenv OMPI_MCA_coll_tuned_scatter_algorithm 2 +setenv OMPI_MCA_coll_tuned_reduce_scatter_algorithm 3 +setenv OMPI_MCA_coll_tuned_allreduce_algorithm 3 +setenv OMPI_MCA_coll_tuned_allgather_algorithm 4 +setenv OMPI_MCA_coll_tuned_allgatherv_algorithm 3 +setenv OMPI_MCA_coll_tuned_gather_algorithm 1 +setenv OMPI_MCA_coll_tuned_barrier_algorithm 0 +# required for a tuned flag to be effective +setenv OMPI_MCA_coll_tuned_use_dynamic_rules 1 +# disable file locks +setenv OMPI_MCA_sharedfp "^lockedfile,individual" EOF # The below settings seem to be recommended for hybrid -# systems using MVAPICH2 but could change +# systems using MVAPICH but could change -else if( $MPI == mvapich2 ) then - -if( $GPU == "TRUE" ) then -cat > $HOMDIR/SETENV.commands << EOF - setenv MV2_ENABLE_AFFINITY 0 - setenv MV2_RNDV_PROTOCOL RPUT - setenv MV2_USE_RDMA_ONE_SIDED 1 - setenv SLURM_DISTRIBUTION block - setenv MV2_MPIRUN_TIMEOUT 100 - setenv MV2_GATHERV_SSEND_THRESHOLD 256 -EOF +else if( $MPI == mvapich ) then -else +# MVAPICH and GEOS has issues with restart writing. Having the +# oserver write them seems to...work +set RESTART_BY_OSERVER = YES cat > $HOMDIR/SETENV.commands << EOF - setenv MV2_ENABLE_AFFINITY 0 - setenv SLURM_DISTRIBUTION block - setenv MV2_MPIRUN_TIMEOUT 100 - setenv MV2_GATHERV_SSEND_THRESHOLD 256 +setenv MV2_ENABLE_AFFINITY 0 +setenv SLURM_DISTRIBUTION block +setenv MV2_MPIRUN_TIMEOUT 100 +setenv MV2_GATHERV_SSEND_THRESHOLD 256 EOF -endif # if GPU and mvapich2 - else if( $MPI == mpt ) then cat > $HOMDIR/SETENV.commands << EOF - setenv MPI_COLL_REPRODUCIBLE - setenv SLURM_DISTRIBUTION block +setenv MPI_COLL_REPRODUCIBLE +setenv SLURM_DISTRIBUTION block + +#setenv MPI_DISPLAY_SETTINGS 1 +#setenv MPI_VERBOSE 1 - #setenv MPI_DISPLAY_SETTINGS 1 - #setenv MPI_VERBOSE 1 - - unsetenv MPI_MEMMAP_OFF - unsetenv MPI_NUM_MEMORY_REGIONS - setenv MPI_XPMEM_ENABLED yes - unsetenv SUPPRESS_XPMEM_TRIM_THRESH +setenv MPI_MEMMAP_OFF +unsetenv MPI_NUM_MEMORY_REGIONS +setenv MPI_XPMEM_ENABLED yes +unsetenv SUPPRESS_XPMEM_TRIM_THRESH - setenv MPI_LAUNCH_TIMEOUT 40 +setenv MPI_LAUNCH_TIMEOUT 40 - # For some reason, PMI_RANK is randomly set and interferes - # with binarytile.x and other executables. - unsetenv PMI_RANK +setenv MPI_COMM_MAX 1024 +setenv MPI_GROUP_MAX 1024 +setenv MPI_BUFS_PER_PROC 256 - # Often when debugging on MPT, the traceback from Intel Fortran - # is "absorbed" and only MPT's errors are displayed. To allow the - # compiler's traceback to be displayed, uncomment this environment - # variable - #setenv FOR_IGNORE_EXCEPTIONS false +# For some reason, PMI_RANK is randomly set and interferes +# with binarytile.x and other executables. +unsetenv PMI_RANK + +# Often when debugging on MPT, the traceback from Intel Fortran +# is "absorbed" and only MPT's errors are displayed. To allow the +# compiler's traceback to be displayed, uncomment this environment +# variable +#setenv FOR_IGNORE_EXCEPTIONS false EOF else if( $MPI == intelmpi ) then cat > $HOMDIR/SETENV.commands << EOF -setenv I_MPI_DAPL_UD enable setenv I_MPI_ADJUST_ALLREDUCE 12 setenv I_MPI_ADJUST_GATHERV 3 @@ -1696,6 +1674,36 @@ setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs setenv ROMIO_FSTYPE_FORCE "gpfs:" EOF +# Testing by Bill Putman found these to be +# useful flags with Intel MPI on SLES15 on the +# Milan nodes. +# Note 1: Testing by NCCS shows the PSM3 provider +# runs on the Infiniband fabric. Tests show it runs +# up to C720. +# Note 2: When the Cascade Lakes are moved to +# SLES15, these will need to be Milan-only flags +# as Intel MPI will probably work just fine with +# Intel chips. +if ("$BUILT_ON_SLES15" == "TRUE") then +cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_FALLBACK 0 +setenv I_MPI_FABRICS ofi +setenv I_MPI_OFI_PROVIDER psm3 +setenv I_MPI_ADJUST_SCATTER 2 +setenv I_MPI_ADJUST_SCATTERV 2 +setenv I_MPI_ADJUST_GATHER 2 +setenv I_MPI_ADJUST_GATHERV 3 +setenv I_MPI_ADJUST_ALLGATHER 3 +setenv I_MPI_ADJUST_ALLGATHERV 3 +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_REDUCE 10 +setenv I_MPI_ADJUST_BCAST 11 +setenv I_MPI_ADJUST_REDUCE_SCATTER 4 +setenv I_MPI_ADJUST_BARRIER 9 +EOF + +endif # if SLES15 + endif # if NCCS endif # if mpi @@ -1719,8 +1727,8 @@ if ( $SITE == 'NCCS' ) then cat >> $HOMDIR/GPUSTART.commands << _EOF1_ -setenv GPU_NODELIST \`scontrol show hostnames\` -setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$SLURM_JOBID +setenv GPU_NODELIST \`scontrol show hostnames\` +setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$SLURM_JOBID _EOF1_ @@ -1728,31 +1736,31 @@ else if ( $SITE == 'NAS' ) then cat >> $HOMDIR/GPUSTART.commands << _EOF1_ -setenv GPU_NODELIST \`cat \$PBS_NODEFILE | uniq\` -setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$PBS_JOBID +setenv GPU_NODELIST \`cat \$PBS_NODEFILE | uniq\` +setenv CUDA_TMPDIR /tmp/nvidia-hyperq-\$PBS_JOBID _EOF1_ endif # if site cat >> $HOMDIR/GPUSTART.commands << _EOF2_ -setenv CUDA_VISIBLE_DEVICES 0 -setenv CUDA_MPS_CLIENT 1 -setenv CUDA_MPS_PIPE_DIRECTORY \$CUDA_TMPDIR/mps_0 -setenv CUDA_MPS_LOG_DIRECTORY \$CUDA_TMPDIR/mps_log_0 - -foreach node (\$GPU_NODELIST) - ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" -end - -sleep 3 - -foreach node (\$GPU_NODELIST) - echo "Running MPS Server on \$node" - ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/run_mps_server.bash" -end - -sleep 3 +setenv CUDA_VISIBLE_DEVICES 0 +setenv CUDA_MPS_CLIENT 1 +setenv CUDA_MPS_PIPE_DIRECTORY \$CUDA_TMPDIR/mps_0 +setenv CUDA_MPS_LOG_DIRECTORY \$CUDA_TMPDIR/mps_log_0 + +foreach node (\$GPU_NODELIST) + ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" +end + +sleep 3 + +foreach node (\$GPU_NODELIST) + echo "Running MPS Server on \$node" + ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/run_mps_server.bash" +end + +sleep 3 _EOF2_ @@ -1760,9 +1768,9 @@ cat > $HOMDIR/GPUEND.commands << _EOF3_ # Remove the MPS Server on Each GPU Node # -------------------------------------- -foreach node (\$GPU_NODELIST) - ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" -end +foreach node (\$GPU_NODELIST) + ssh -f \$node "env CUDA_TMPDIR=\$CUDA_TMPDIR $HOME/bin/kill_mps_server.bash" +end _EOF3_ @@ -1912,7 +1920,7 @@ s/@GEOSCTM_IM/$GEOSCTM_IM/g s/@GEOSCTM_JM/$GEOSCTM_JM/g s/@GEOSCTM_LM/$GEOSCTM_LM/g s/@BEG_DATE/${BEG_DATE}/g -s/@END_DATE/${END_DATE}/g +s/@END_DATE/${END_DATE}/g s/@JOB_SGMT/${JOB_SGMT}/g s/@NUM_SGMT/${NUM_SGMT}/g @@ -2128,7 +2136,7 @@ endif /bin/mv $HOMDIR/pTracers_GridComp.rc.tmpl $HOMDIR/pTracers_GridComp.rc ####################################################################### -# Modify RC Directory for LM and PCHEM/GOCART/GOCART.data Options +# Modify RC Directory for LM and PCHEM/GOCART/GOCART.data Options ####################################################################### # Modify RC Files for LM @@ -2370,7 +2378,7 @@ else if ( $ctmGMI == TRUE ) then sed -e 's|EXTDATA|/discover/nobackup/projects/gmao/share/dasilva/fvInput|g' $EXPDIR/RC/GMI_GridComp.rc > $EXPDIR/RC/GMI_GridComp.tmp /bin/mv -f $EXPDIR/RC/GMI_GridComp.tmp $EXPDIR/RC/GMI_GridComp.rc endif - + # Turn on GEOS-Chem # ----------- else if ( $ctmGEOS_Chem == TRUE ) then @@ -2555,150 +2563,31 @@ if( -e $HOMDIR/tmpfile ) /bin/rm $HOMDIR/tmpfile if( -e $HOMDIR/sedfile ) /bin/rm $HOMDIR/sedfile ####################################################################### -# Determine Experiment Specific src Files -####################################################################### - -if ( $NOCVS != "TRUE" ) then - -echo "ERROR! Repository managment is not supported yet in this model" -echo " due to the move to git. If this section is reached," -echo " something has gone wrong. Please contact Matt Thompson" -echo " or Larry Takacs" -exit 3 - -# Make a src directory under EXPDIR to hold current Experiment files -# Note: Sandbox Source Location: $GEOSDIR/src -# EXP TAG Source Location: $EXPDIR/src/GEOSctm/src -# ------------------------------------------------------------------ -/bin/rm -rf ${EXPDIR}/src -mkdir -p ${EXPDIR}/src/GEOSctm/src -cd ${EXPDIR}/src/GEOSctm/src - -echo "${C2}Copying${CN} ${C1}Sandbox Source Code${CN} ${C2}(including non-committed files) into" -echo " ${CN} ${C1}${EXPDIR}/src/GEOSctm/src${CN} ${C2}...${CN}" -# ---------------------------------------------------------------------------------------------------------------------------------- -rsync -ar --exclude '*.o' \ - --exclude '*.x' \ - --exclude '*.xx' \ - --exclude '*.a' \ - --exclude '*.d' \ - --exclude '*.nc4' \ - --exclude '*.mod' \ - --exclude 'GEOSctm.x.*' \ - ${GEOSDIR}/src/* . -echo " " - -# ----------------------------------------------------------------------------- -# Create CVS TAG for Experiment -# ----------------------------------------------------------------------------- - -# Change Dots to Underscores in EXPID for CVS Tag -# ----------------------------------------------- -set EXPIDTAG = `echo ${EXPID} | sed -e 's/\./_/g'` -set tagname = "${EXPIDTAG}__${LOGNAME}" - -if( ! $?ESMA_NOCOMMIT ) then -TAGNAME: - echo "Enter ${C1}CVS Tag${CN} to COMMIT this Experiment (Default: ${C2}${tagname}${CN})" - echo "Enter ${C1}q${CN} or ${C1}quit${CN} to QUIT or SKIP the CVS COMMIT:" - set TAGTMP = $< - if ( .$TAGTMP != . ) then - set tagname = "$TAGTMP" - endif - # Change Dots to Underscores in TAGNAME for CVS Tag - # ------------------------------------------------- - set tagname = `echo ${tagname} | sed -e 's/\./_/g'` - if( $tagname == 'q' | $tagname == 'quit' ) then - setenv ESMA_NOCOMMIT TRUE - set tagname = "${EXPIDTAG}__${LOGNAME}" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_FALSE - else - set tagtest = `cvs status -v g5_modules | grep ${tagname}` - if( $#tagtest != 0 ) then - echo " ${C1}CVS COMMIT Tag${CN} ${C2}${tagname}${CN} already exists." - set tagname = "${EXPIDTAG}__${LOGNAME}" - goto TAGNAME - endif - endif -else - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_FALSE -endif - -/bin/rm -f ${EXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION -echo $tagname > ${EXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION - - -# Create a list of non-committed Sandbox files (ignore updated files not included in sandbox) -# ------------------------------------------------------------------------------------------- -cvs -nq upd | grep -v "U " > ${EXPDIR}/src/srcfiles - - -echo "${C2}Non-Committed Sandbox Files:${CN}" -echo "---------------------------------" - -/bin/rm -f ${EXPDIR}/src/cvs.log - -while( -e ${EXPDIR}/src/srcfiles ) - set file = `head -1 ${EXPDIR}/src/srcfiles` - set bit = `echo "$file" | cut -c1` - if( "$bit" != "?" ) then - set file = `echo "$file" | cut -d' ' -f2` - echo $file +# Copy over Source Tarfile +####################################################################### - # Local Copy of Differing Sandbox File from TAG file - # -------------------------------------------------- - set newfile = `echo $file | sed -e "s?/?^?g"` - /bin/cp -f ${EXPDIR}/src/GEOSctm/src/$file ${EXPDIR}/src/$newfile +# NOTE: This variable is set at CMake time depending on +# how the build was configured. +set INSTALL_TARFILE = @CFG_INSTALL_SOURCE_TARFILE@ +set TARFILE_NAME = "@CMAKE_PROJECT_NAME@.tar.gz" - # CVS Commit updated Sandbox files - # -------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - set branchname = "BRANCH__${tagname}" - cvs tag -b ${branchname} $file >& ${EXPDIR}/src/cvs.log - cvs upd -r ${branchname} $file >& ${EXPDIR}/src/cvs.log - cvs commit -m "${tagname} ${EXPDSC}" $file >& ${EXPDIR}/src/cvs.log - endif +if ( $INSTALL_TARFILE == "TRUE" ) then - endif - sed 1,1d -i ${EXPDIR}/src/srcfiles - set nfiles = `wc -c ${EXPDIR}/src/srcfiles | cut -d" " -f 1` - if( $nfiles == 0 ) /bin/rm -f ${EXPDIR}/src/srcfiles -end - -echo " " - # Tag Final EXPDIR/src/GEOSctm/src - # --------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - cvs tag ${tagname} >& ${EXPDIR}/src/cvs.log - if( $status == 0 ) then - echo "${C2}EXP:${CN} ${C1}$EXPID${CN} ${C2}successfully committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_TRUE - else - echo "${C2}EXP:${CN} ${C1}$EXPID${CN} ${C2}failed to be committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${EXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${EXPDIR}/src/COMMIT_STATUS_* - touch ${EXPDIR}/src/COMMIT_STATUS_FALSE - endif - endif - -if( -e ${EXPDIR}/src/cvs.log ) /bin/rm -f ${EXPDIR}/src/cvs.log -cd ${EXPDIR}/src + # Make a src directory under EXPDIR to hold current Experiment files + # ------------------------------------------------------------------ + /bin/rm -rf ${EXPDIR}/src + mkdir -p ${EXPDIR}/src -echo "" -echo "${C2}Tarring${CN} Experiment Source Code into Single File ${C2}...${CN}" -# --------------------------------------------------------------------------- -/bin/tar cf ${EXPID}.GEOSctm.tar GEOSctm -/bin/rm -r GEOSctm -echo "" + echo "Copying Build Source Code into ${C2}${EXPDIR}/src${CN}" + # ----------------------------------------------------------- + if (-e ${GEOSDEF}/src/${TARFILE_NAME}) then + cp ${GEOSDEF}/src/${TARFILE_NAME} ${EXPDIR}/src + else + echo "${GEOSDEF}/src/${TARFILE_NAME} not found yet CMake was asked to make and install a tarfile." + echo "Something went wrong." + exit 7 + endif + echo "" endif @@ -2730,7 +2619,7 @@ else if ( ! -d $CLONEDIR ) then endif # ------------------------------------------------------ -# To setup the clone, we need to look in a couple files, +# To setup the clone, we need to look in a couple files, # so make sure they are readable # ------------------------------------------------------ @@ -2878,7 +2767,7 @@ EOF # ------------------------------------------------ # Create two sets of files so we can copy from one -# directory to another. +# directory to another. # ------------------------------------------------ # Then alter them @@ -3007,149 +2896,32 @@ echo "" /bin/rm $COPYSCRIPT /bin/rm $SEDFILE -# -------------------------------- -# Cloned Experiment Source Control -# -------------------------------- - -if ( $NOCVS != "TRUE" ) then - -echo "ERROR! Repository managment is not supported yet in this model" -echo " due to the move to git. If this section is reached," -echo " something has gone wrong. Please contact Matt Thompson" -echo " or Larry Takacs" -exit 3 - -# Make a src directory under NEWEXPDIR to hold current Experiment files -# Note: Sandbox Source Location: $GEOSDIR/src -# EXP TAG Source Location: $NEWEXPDIR/src/GEOSctm/src -# ------------------------------------------------------------------ -/bin/rm -rf ${NEWEXPDIR}/src -mkdir -p ${NEWEXPDIR}/src/GEOSctm/src -cd ${NEWEXPDIR}/src/GEOSctm/src - -echo "${C2}Copying${CN} ${C1}Sandbox Source Code${CN} ${C2}(including non-committed files) into${CN} ${C1}${NEWEXPDIR}/src/GEOSctm/src${CN} ${C2}...${CN}" -# --------------------------------------------------------------------------------------------------------------------------------- -rsync -ar --exclude '*.o' \ - --exclude '*.x' \ - --exclude '*.xx' \ - --exclude '*.a' \ - --exclude '*.d' \ - --exclude '*.nc4' \ - --exclude '*.mod' \ - --exclude 'GEOSctm.x.*' \ - ${GEOSDIR}/src/* . -echo " " - -# ----------------------------------------------------------------------------- -# Create CVS TAG for Experiment -# ----------------------------------------------------------------------------- - -# Change Dots to Underscores in NEWEXPID for CVS Tag -# -------------------------------------------------- -set EXPIDTAG = `echo ${NEWEXPID} | sed -e 's/\./_/g'` -set tagname = "${EXPIDTAG}__${LOGNAME}" - -if( ! $?ESMA_NOCOMMIT ) then -NEWTAGNAME: - echo "Enter ${C1}CVS Tag${CN} to COMMIT this Experiment (Default: ${C2}${tagname}${CN})" - echo "Enter ${C1}q${CN} or ${C1}quit${CN} to QUIT or SKIP the CVS COMMIT:" - set TAGTMP = $< - if ( .$TAGTMP != . ) then - set tagname = "$TAGTMP" - endif - # Change Dots to Underscores in TAGNAME for CVS Tag - # ------------------------------------------------- - set tagname = `echo ${tagname} | sed -e 's/\./_/g'` - if( $tagname == 'q' | $tagname == 'quit' ) then - setenv ESMA_NOCOMMIT TRUE - set tagname = "${EXPIDTAG}__${LOGNAME}" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_FALSE - else - set tagtest = `cvs status -v g5_modules | grep ${tagname}` - if( $#tagtest != 0 ) then - echo " ${C1}CVS COMMIT Tag${CN} ${C2}${tagname}${CN} already exists." - set tagname = "${EXPIDTAG}__${LOGNAME}" - goto NEWTAGNAME - endif - endif -else - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_FALSE -endif - -/bin/rm -f ${NEWEXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION -echo $tagname > ${NEWEXPDIR}/src/GEOSctm/src/Applications/GEOSctm_App/.GEOSCTM_VERSION - - -# Create a list of non-committed Sandbox files -# -------------------------------------------- -cvs -nq upd > ${NEWEXPDIR}/src/srcfiles - - -echo "${C2}Non-Committed Sandbox Files:${CN}" -echo "---------------------------------" - -/bin/rm -f ${NEWEXPDIR}/src/cvs.log - -while( -e ${NEWEXPDIR}/src/srcfiles ) - set file = `head -1 ${NEWEXPDIR}/src/srcfiles` - set bit = `echo "$file" | cut -c1` - if( "$bit" != "?" ) then - set file = `echo "$file" | cut -d' ' -f2` - echo $file +# ------------------------ +# Cloned Experiment Source +# ------------------------ - # Local Copy of Differing Sandbox File from TAG file - # -------------------------------------------------- - set newfile = `echo $file | sed -e "s?/?^?g"` - /bin/cp -f ${NEWEXPDIR}/src/GEOSctm/src/$file ${NEWEXPDIR}/src/$newfile +# NOTE: This variable is set at CMake time depending on +# how the build was configured. +set INSTALL_TARFILE = @CFG_INSTALL_SOURCE_TARFILE@ +set TARFILE_NAME = "@CMAKE_PROJECT_NAME@.tar.gz" - # CVS Commit updated Sandbox files - # -------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - set branchname = "BRANCH__${tagname}" - cvs tag -b ${branchname} $file >& ${NEWEXPDIR}/src/cvs.log - cvs upd -r ${branchname} $file >& ${NEWEXPDIR}/src/cvs.log - cvs commit -m "${tagname} ${EXPDSC}" $file >& ${NEWEXPDIR}/src/cvs.log - endif +if ( $INSTALL_TARFILE == "TRUE" ) then - endif - sed 1,1d -i ${NEWEXPDIR}/src/srcfiles - set nfiles = `wc -c ${NEWEXPDIR}/src/srcfiles | cut -d" " -f 1` - if( $nfiles == 0 ) /bin/rm -f ${NEWEXPDIR}/src/srcfiles -end + # Make a src directory under EXPDIR to hold current Experiment files + # ------------------------------------------------------------------ + /bin/rm -rf ${NEWEXPDIR}/src + mkdir -p ${NEWEXPDIR}/src -echo " " - # Tag Final NEWEXPDIR/src/GEOSctm/src - # ----------------------------------- - if( ! $?ESMA_NOCOMMIT ) then - cvs tag ${tagname} >& ${NEWEXPDIR}/src/cvs.log - if( $status == 0 ) then - echo "${C2}EXP:${CN} ${C1}$NEWEXPID${CN} ${C2}successfully committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_TRUE - else - echo "${C2}EXP:${CN} ${C1}$NEWEXPID${CN} ${C2}failed to be committed with TAG:${CN} ${C1}${tagname}${CN}" - echo "-------------------------------------------------------" - set NUM_COMMIT_STATUS = `/bin/ls -1 ${NEWEXPDIR}/src | grep COMMIT_STATUS` - if($#NUM_COMMIT_STATUS > 0 ) /bin/rm -f ${NEWEXPDIR}/src/COMMIT_STATUS_* - touch ${NEWEXPDIR}/src/COMMIT_STATUS_FALSE - endif - endif - -/bin/rm -f ${NEWEXPDIR}/src/cvs.log -cd ${NEWEXPDIR}/src - -echo "" -echo "${C2}Tarring${CN} Experiment Source Code into Single File ${C2}...${CN}" -# --------------------------------------------------------------------------- - tar cf ${NEWEXPID}.GEOSctm.tar GEOSctm -/bin/rm -r GEOSctm -echo "" + echo "Copying Build Source Code into ${C2}${NEWEXPDIR}/src${CN}" + # -------------------------------------------------------------- + if (-e ${GEOSDEF}/src/${TARFILE_NAME}) then + cp ${GEOSDEF}/src/${TARFILE_NAME} ${NEWEXPDIR}/src + else + echo "${GEOSDEF}/src/${TARFILE_NAME} not found yet CMake was asked to make and install a tarfile." + echo "Something went wrong." + exit 7 + endif + echo "" endif @@ -3176,8 +2948,8 @@ cat << EOF ctm_setup requires access to internet resources, namely the CVS repository. You are attempting to run ctm_setup -on compute node ${NODE} at ${SITE}. Please re-run on a head -node, or supply the --nocvs option. +on compute node ${NODE} at ${SITE}. Please re-run on a head +node, or supply the --nocvs option. For more information, contact Matt Thompson or Larry Takacs at GMAO. @@ -3190,9 +2962,9 @@ CONTACTMATT: cat < Date: Tue, 20 Aug 2024 08:01:43 -0400 Subject: [PATCH 2/6] Update codeowners --- .github/CODEOWNERS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 60263a3..2abc961 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,16 +3,16 @@ # Order is important; the last matching pattern takes the most # precedence. -# Everything in GEOSgcm should be owned by the GCM Gatekeepers +# Everything in GEOSctm should be owned by the CTM Gatekeepers * @GEOS-ESM/ctm-gatekeepers -# The GCM gatekeepers and CMake should know/approve these +# The CTM gatekeepers and CMake should know/approve these /.github/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers -/.circleci/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers -/.codebuild/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers +/.circleci/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers +/.codebuild/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers -# The GEOS CMake Team is the CODEOWNER for the CMakeLists.txt files in this repository -CMakeLists.txt @GEOS-ESM/cmake-team +# The GEOS CMake Team should be notified about changes to the CMakeLists.txt files in this repository +CMakeLists.txt @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers # The GEOS CMake Team should be notified about and approve config changes -/config/ @GEOS-ESM/cmake-team +/config/ @GEOS-ESM/cmake-team @GEOS-ESM/ctm-gatekeepers From e9707a97c0e764a090a1cf8480b4432fc793ed46 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 20 Aug 2024 08:05:35 -0400 Subject: [PATCH 3/6] Few more updates --- src/Applications/GEOSctm_App/CMakeLists.txt | 14 ++- src/Applications/GEOSctm_App/ctm_convert.j | 98 ++++++++--------- src/Applications/GEOSctm_App/ctm_regress.j | 114 ++++++++++---------- 3 files changed, 112 insertions(+), 114 deletions(-) diff --git a/src/Applications/GEOSctm_App/CMakeLists.txt b/src/Applications/GEOSctm_App/CMakeLists.txt index b86c49f..826a0d0 100644 --- a/src/Applications/GEOSctm_App/CMakeLists.txt +++ b/src/Applications/GEOSctm_App/CMakeLists.txt @@ -42,13 +42,11 @@ else() set(CFG_HYDROSTATIC FALSE) endif() -# Did we build for AMD Rome hardware (aka EPYC)? -cmake_host_system_information(RESULT proc_decription QUERY PROCESSOR_DESCRIPTION) -if (${proc_decription} MATCHES "EPYC") - set(CFG_BUILT_ON_ROME TRUE) -else () - set(CFG_BUILT_ON_ROME FALSE) -endif () +if(INSTALL_SOURCE_TARFILE) + set(CFG_INSTALL_SOURCE_TARFILE TRUE) +else() + set(CFG_INSTALL_SOURCE_TARFILE FALSE) +endif() set (setup_scripts ctm_setup @@ -59,6 +57,6 @@ foreach (file ${setup_scripts}) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${file} DESTINATION bin) endforeach () - configure_file(.GEOSCTM_VERSION .GEOSCTM_VERSION @ONLY) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/.GEOSCTM_VERSION DESTINATION etc) + diff --git a/src/Applications/GEOSctm_App/ctm_convert.j b/src/Applications/GEOSctm_App/ctm_convert.j index 9fa4727..af2c253 100755 --- a/src/Applications/GEOSctm_App/ctm_convert.j +++ b/src/Applications/GEOSctm_App/ctm_convert.j @@ -37,7 +37,7 @@ setenv ARCH `uname` setenv SITE @SITE setenv GEOSDIR @GEOSDIR -setenv GEOSBIN @GEOSBIN +setenv GEOSBIN @GEOSBIN setenv GCMVER @GCMVER echo "Sourcing g5_modules in $GEOSBIN" @@ -85,9 +85,9 @@ set GEOSCTM_LM = `grep GEOSctm_LM: $HOMDIR/GEOSCTM.rc | cut -d':' -f2` set OGCM_IM = `grep "OGCM\.IM_WORLD:" $HOMDIR/GEOSCTM.rc | cut -d':' -f2` set OGCM_JM = `grep "OGCM\.JM_WORLD:" $HOMDIR/GEOSCTM.rc | cut -d':' -f2` ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>FVCUBED<<>>DATAOCEAN<<>>COUPLED<<>>DATAOCEAN<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>DATAOCEAN<<>>OSTIA<< $FILE #!/bin/csh -f ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<< $FILE /bin/ln -sf $BCSDIR/Shared/*bin . /bin/ln -sf $BCSDIR/Shared/*c2l*.nc4 . ->>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>DATAOCEAN<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< $FILE >>>FVCUBED<<>>FVCUBED<<>>COUPLED<<<@CPEXEC $HOMDIR/*_table . ->>>COUPLED<<<@CPEXEC $GRIDDIR/INPUT/* INPUT ->>>COUPLED<<>>COUPLED<<>>COUPLED<<<@CPEXEC $CNVDIR/RESTART/* INPUT +@COUPLED @CPEXEC $CNVDIR/RESTART/* INPUT ####################################################################### # Set Experiment Run Parameters that were altered @@ -519,7 +519,7 @@ if ( $rc != 0 ) then else echo "Conversion successful!" endif - + ####################################################################### # Now copy the checkpoints into $RSTDIR ####################################################################### @@ -527,26 +527,26 @@ endif set numrst = `echo $found_rst_types | wc -w` set numchk = `echo $chk_files | wc -w` -@ n = 1 -@ z = $numchk + 1 +@ n = 1 +@ z = $numchk + 1 while ( $n <= $numrst ) if ( -e $found_rst_files[$n] ) then - @ m = 1 + @ m = 1 while ( $m <= $numchk ) if( $chk_files[$m] == $found_rst_types[$n] || ) then /bin/mv $chk_file_names[$m] $RSTDIR/$found_rst_files[$n]$toext - @ m = $numchk + 999 + @ m = $numchk + 999 else - @ m = $m + 1 + @ m = $m + 1 endif - end + end wait if( $m == $z ) then echo "Warning!! Could not find CHECKPOINT/RESTART match for: " $chk_files[$n] exit endif endif -@ n = $n + 1 +@ n = $n + 1 end # Copy over cap_restart as well diff --git a/src/Applications/GEOSctm_App/ctm_regress.j b/src/Applications/GEOSctm_App/ctm_regress.j index 540f948..f41bf40 100755 --- a/src/Applications/GEOSctm_App/ctm_regress.j +++ b/src/Applications/GEOSctm_App/ctm_regress.j @@ -77,15 +77,15 @@ cd $EXPDIR/regress @CPEXEC $EXPDIR/GEOSctm.x $EXPDIR/regress @CPEXEC $EXPDIR/linkbcs $EXPDIR/regress @CPEXEC $HOMDIR/*.yaml $EXPDIR/regress ->>>COUPLED<<<@CPEXEC $HOMDIR/*.nml $EXPDIR/regress ->>>MOM6<<<@CPEXEC $HOMDIR/MOM_input $EXPDIR/regress ->>>MOM6<<<@CPEXEC $HOMDIR/MOM_override $EXPDIR/regress +@COUPLED @CPEXEC $HOMDIR/*.nml $EXPDIR/regress +@MOM6 @CPEXEC $HOMDIR/MOM_input $EXPDIR/regress +@MOM6 @CPEXEC $HOMDIR/MOM_override $EXPDIR/regress cat fvcore_layout.rc >> input.nml if(-e ExtData.rc ) /bin/rm -f ExtData.rc set extdata_files = `/bin/ls -1 *_ExtData.rc` -cat $extdata_files > ExtData.rc +cat $extdata_files > ExtData.rc # Define Atmospheric Resolution # ----------------------------- @@ -127,8 +127,8 @@ foreach rst ( $rst_file_names ) end @CPEXEC $EXPDIR/cap_restart $EXPDIR/regress ->>>COUPLED<<>>COUPLED<<<@CPEXEC $EXPDIR/RESTART/* INPUT +@COUPLED /bin/mkdir INPUT +@COUPLED @CPEXEC $EXPDIR/RESTART/* INPUT setenv YEAR `cat cap_restart | cut -c1-4` ./linkbcs @@ -163,7 +163,7 @@ endif endif ####################################################################### -# Create Simple History for Efficiency +# Create Simple History for Efficiency ####################################################################### set FILE = HISTORY.rc0 @@ -245,7 +245,7 @@ endif if(-e ExtData.rc ) /bin/rm -f ExtData.rc set extdata_files = `/bin/ls -1 *_ExtData.rc` -cat $extdata_files > ExtData.rc +cat $extdata_files > ExtData.rc # If REPLAY, link necessary forcing files @@ -267,7 +267,7 @@ if( $REPLAY_MODE == 'Exact' | $REPLAY_MODE == 'Regular' ) then /bin/ln -sf ${ANA_LOCATION}/${REPLAY_FILE_TYPE} . /bin/ln -sf ${ANA_LOCATION}/${REPLAY_FILE09_TYPE} . -endif +endif ################################################################## ###### @@ -298,7 +298,7 @@ set NX = `grep "^ *NX": GEOSCTM.rc | cut -d':' -f2` set NY = `grep "^ *NY": GEOSCTM.rc | cut -d':' -f2` @ NPES = $NX * $NY $RUN_CMD $NPES ./GEOSctm.x - + set date = `cat cap_restart` set nymde = $date[1] @@ -307,7 +307,7 @@ set nhmse = $date[2] foreach chk ( $chk_file_names ) /bin/mv $chk ${chk}.${nymde}_${nhmse}.1 end ->>>MOM6<<>>DATAOCEAN<<>>DATAOCEAN<<< @ test_NX = $NPES0 / 6 ->>>DATAOCEAN<<< @ test_NP = $IM / $test_NX ->>>DATAOCEAN<<< if($test_NP < 4 ) then ->>>DATAOCEAN<<< @ test_NX = $IM / 4 # To ensure enough gridpoints for HALO ->>>DATAOCEAN<<< endif ->>>DATAOCEAN<<< set test_NY = 6 ->>>DATAOCEAN<<>>DATAOCEAN<<< set test_NX = $NY0 ->>>DATAOCEAN<<< set test_NY = $NX0 ->>>DATAOCEAN<<>>COUPLED<<>>COUPLED<< cap_restart @@ -388,7 +388,7 @@ while ( $n <= $numchk ) @ n = $n + 1 end ->>>COUPLED<<<@CPEXEC RESTART/* INPUT +@COUPLED @CPEXEC RESTART/* INPUT ################################################################## ###### @@ -423,17 +423,17 @@ set oldstring = `cat GEOSCTM.rc | grep "^ *NY:"` set newstring = "NY: ${test_NY}" /bin/mv GEOSCTM.rc GEOSCTM.tmp cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< GEOSCTM.rc ->>>COUPLED<<>>COUPLED<<>>COUPLED<<>>COUPLED<< GEOSCTM.rc - ->>>MOM5<<>>MOM6<< GEOSCTM.rc +@COUPLED set oldstring = `cat GEOSCTM.rc | grep "^ *OGCM.NY:"` +@COUPLED set newstring = "OGCM.NY: ${test_NX}" +@COUPLED /bin/mv GEOSCTM.rc GEOSCTM.tmp +@COUPLED cat GEOSCTM.tmp | sed -e "s?$oldstring?$newstring?g" > GEOSCTM.rc + +@MOM5 sed -r -i -e "/^ *layout/ s#= ([0-9]+),*([0-9]+)#= ${test_NY},${test_NX}#" input.nml +@MOM6 sed -r -i -e "/^ *LAYOUT/ s#= ([0-9]+), *([0-9]+)#= ${test_NY}, ${test_NX}#" MOM_input setenv YEAR `cat cap_restart | cut -c1-4` ./linkbcs @@ -441,7 +441,7 @@ set NX = `grep "^ *NX": GEOSCTM.rc | cut -d':' -f2` set NY = `grep "^ *NY": GEOSCTM.rc | cut -d':' -f2` @ NPES = $NX * $NY $RUN_CMD $NPES ./GEOSctm.x - + set date = `cat cap_restart` set nymde = $date[1] set nhmse = $date[2] @@ -449,7 +449,7 @@ set nhmse = $date[2] foreach chk ( $chk_file_names ) /bin/mv $chk ${chk}.${nymde}_${nhmse}.2 end ->>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<>>MOM6<<< set check = true ->>>MOM6<<< if( $check == true ) then ->>>MOM6<<< echo Comparing "MOM6 restarts" ->>>MOM6<<< cmp $file1 $file2 ->>>MOM6<<< if( $status == 0 ) then ->>>MOM6<<< echo Success! ->>>MOM6<<< echo " " ->>>MOM6<<< else ->>>MOM6<<< echo Failed! ->>>MOM6<<< echo " " ->>>MOM6<<< set pass = false ->>>MOM6<<< endif ->>>MOM6<<< endif ->>>MOM6<< Date: Tue, 20 Aug 2024 08:06:59 -0400 Subject: [PATCH 4/6] Add GEOS_Util --- components.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components.yaml b/components.yaml index c830dfc..d0ee5f0 100644 --- a/components.yaml +++ b/components.yaml @@ -33,6 +33,12 @@ GMAO_Shared: sparse: ./config/GMAO_Shared.sparse develop: main +GEOS_Util: + local: ./src/Shared/@GMAO_Shared/@GEOS_Util + remote: ../GEOS_Util.git + tag: v2.1.2 + develop: main + MAPL: local: ./src/Shared/@MAPL remote: ../MAPL.git From d154c990dfc69d1ff58b56535876ab40123b76dc Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 20 Aug 2024 08:09:03 -0400 Subject: [PATCH 5/6] Remove NCEP_Shared --- components.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/components.yaml b/components.yaml index d0ee5f0..01de000 100644 --- a/components.yaml +++ b/components.yaml @@ -19,12 +19,6 @@ ecbuild: remote: ../ecbuild.git tag: geos/v1.3.0 -NCEP_Shared: - local: ./src/Shared/@NCEP_Shared - remote: ../NCEP_Shared.git - tag: v1.3.0 - sparse: ./config/NCEP_Shared.sparse - develop: main GMAO_Shared: local: ./src/Shared/@GMAO_Shared From 15458ff51fad7cea9b8be104b96f0a7fed0a780f Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Tue, 20 Aug 2024 08:12:58 -0400 Subject: [PATCH 6/6] Add presets --- presets/CMakeDefaultPresets.json | 13 +++++++++++++ presets/CMakeNCCSPresets.json | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 presets/CMakeDefaultPresets.json create mode 100644 presets/CMakeNCCSPresets.json diff --git a/presets/CMakeDefaultPresets.json b/presets/CMakeDefaultPresets.json new file mode 100644 index 0000000..3dbaf9e --- /dev/null +++ b/presets/CMakeDefaultPresets.json @@ -0,0 +1,13 @@ +{ + "configurePresets": [ + { + "name": "base-configure", + "hidden": true, + "displayName": "Base Configure Settings", + "description": "Sets build and install directories", + "binaryDir": "${sourceDir}/build-${presetName}", + "installDir": "${sourceDir}/install-${presetName}" + } + ], + "version": 7 +} diff --git a/presets/CMakeNCCSPresets.json b/presets/CMakeNCCSPresets.json new file mode 100644 index 0000000..4902c9b --- /dev/null +++ b/presets/CMakeNCCSPresets.json @@ -0,0 +1,13 @@ +{ + "configurePresets": [ + { + "name": "base-configure", + "hidden": true, + "displayName": "Base Configure Settings", + "description": "Sets build and install directories", + "binaryDir": "$penv{CMAKE_BUILD_LOCATION}/${sourceDirName}/build-${presetName}", + "installDir": "$penv{CMAKE_INSTALL_LOCATION}/${sourceDirName}/install-${presetName}" + } + ], + "version": 7 +}