diff --git a/.github/inactive_workflows/fesom2_icepack.yml b/.github/inactive_workflows/fesom2_icepack.yml index 9057057b1..e1f34568e 100644 --- a/.github/inactive_workflows/fesom2_icepack.yml +++ b/.github/inactive_workflows/fesom2_icepack.yml @@ -14,6 +14,9 @@ jobs: # Docker Hub image that `container-job` executes in container: koldunovn/fesom2_test:refactoring2 + env: + CMAKE_ARGS: -DUSE_ICEPACK=ON + # Service containers to run with `gfortran_ubuntu` steps: # NK: this changes working directory to fesom2 @@ -21,7 +24,6 @@ jobs: - name: prepeare FESOM2_icepack run: | - sed -i 's/USE_ICEPACK OFF/USE_ICEPACK ON/g' CMakeLists.txt cd ./src/icepack_drivers/ ./download_icepack.sh cd ../../ diff --git a/.github/workflows/fesom2.1.yml b/.github/workflows/fesom2.1.yml index 069a5a669..21183e5b1 100644 --- a/.github/workflows/fesom2.1.yml +++ b/.github/workflows/fesom2.1.yml @@ -19,17 +19,18 @@ jobs: # NK: this changes working directory to fesom2 - uses: actions/checkout@v2 - - name: Compile model (binary) + - name: Compile model run: | bash -l configure.sh ubuntu - - name: Compile model (library) + - name: Check library exists run: | - bash ./test/ifs_interface/configure_lib.sh -l + ./test/ifs_interface/check_exist.sh - - name: Library exists + - name: Check downstream find_package(fesom) run: | - bash ./test/ifs_interface/check_exist.sh + cd test_downstream + cmake -S . -B build -Dfesom_ROOT=$(pwd)/.. - name: Create global test run run: | diff --git a/.github/workflows/fesom2_openmp.yml b/.github/workflows/fesom2_openmp.yml index fd1e8cafa..7f6294c22 100644 --- a/.github/workflows/fesom2_openmp.yml +++ b/.github/workflows/fesom2_openmp.yml @@ -14,17 +14,14 @@ jobs: # Docker Hub image that `container-job` executes in container: koldunovn/fesom2_test:refactoring2 + env: + CMAKE_ARGS: -DENABLE_OPENMP=ON + # Service containers to run with `gfortran_ubuntu` steps: # NK: this changes working directory to fesom2 - uses: actions/checkout@v2 - - name: switch OpenMP ON - run: | - cd ./src/ - sed -i 's/with OpenMP\" OFF/with OpenMP\" ON/g' CMakeLists.txt - cd ../ - - name: Compile model run: | bash -l configure.sh ubuntu diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ec4985d5..a623f3df8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,4 +17,4 @@ set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical #set(VERBOSE OFF CACHE BOOL "toggle debug output") #add_subdirectory(oasis3-mct/lib/psmile) -add_subdirectory(src) +add_subdirectory(src fesom) diff --git a/cmake/FindNETCDF.cmake b/cmake/FindNETCDF.cmake index 03b985d67..a2b40feb1 100644 --- a/cmake/FindNETCDF.cmake +++ b/cmake/FindNETCDF.cmake @@ -13,9 +13,13 @@ if(CMAKE_Fortran_COMPILER_LOADED) if(HAVE_Fortran_NETCDF) set(NETCDF_Fortran_INCLUDE_DIRECTORIES "") set(NETCDF_Fortran_LIBRARIES "") + set(NETCDF_Fortran_FOUND 1) else() - find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) + find_path(NETCDF_Fortran_INCLUDE_DIRECTORIES netcdf.inc HINTS $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_Fortran_INCLUDE_DIRECTORIES) find_library(NETCDF_Fortran_LIBRARIES netcdff HINTS ${NETCDF_Fortran_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_Fortran_INCLUDE_DIRECTORIES AND NETCDF_Fortran_LIBRARIES ) + set(NETCDF_Fortran_FOUND 1) + endif() endif() endif() @@ -27,14 +31,18 @@ if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED) if(HAVE_C_NETCDF) set(NETCDF_C_INCLUDE_DIRECTORIES "") set(NETCDF_C_LIBRARIES "") + set(NETCDF_C_FOUND 1) else() - find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) + find_path(NETCDF_C_INCLUDE_DIRECTORIES netcdf.h HINTS $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR}/include ENV NETCDF_C_INCLUDE_DIRECTORIES) find_library(NETCDF_C_LIBRARIES netcdf HINTS ${NETCDF_C_INCLUDE_DIRECTORIES}/../lib) + if( NETCDF_C_INCLUDE_DIRECTORIES AND NETCDF_C_LIBRARIES ) + set(NETCDF_C_FOUND 1) + endif() endif() endif() if(CMAKE_CXX_COMPILER_LOADED) - find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_DIR}/include ENV NETCDF_CXX_INCLUDE_DIRECTORIES) + find_path(NETCDF_CXX_INCLUDE_DIRECTORIES netcdf HINTS $ENV{NETCDF_DIR}/include $ENV{NETCDF4_DIR} ENV NETCDF_CXX_INCLUDE_DIRECTORIES) # the cray toolchain (e.g. hlrn) disables dynamic linking by default. to enable it at build time do e.g. "CRAYPE_LINK_TYPE=dynamic make". find_library(NETCDF_CXX_LIBRARIES NAMES netcdf_c++4 netcdf-cxx4 HINTS ${NETCDF_CXX_INCLUDE_DIRECTORIES}/../lib) if(NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_C_INCLUDE_DIRECTORIES) @@ -43,4 +51,21 @@ if(CMAKE_CXX_COMPILER_LOADED) if(NETCDF_CXX_LIBRARIES AND NETCDF_C_LIBRARIES) list(APPEND NETCDF_CXX_LIBRARIES ${NETCDF_C_LIBRARIES}) endif() -endif() \ No newline at end of file + if( NETCDF_CXX_INCLUDE_DIRECTORIES AND NETCDF_CXX_LIBRARIES ) + set(NETCDF_CXX_FOUND 1) + endif() +endif() + +if(NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + set(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS C) +endif() + +unset({CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) +foreach(COMP C CXX Fortran) + if("${COMP}" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + list(APPEND ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS NETCDF_${COMP}_FOUND) + endif() +endforeach() +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} HANDLE_COMPONENTS REQUIRED_VARS ${CMAKE_FIND_PACKAGE_NAME}_REQUIRED_VARS) + diff --git a/configure.sh b/configure.sh index b4b6e27b8..ff4482275 100755 --- a/configure.sh +++ b/configure.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -e - +HERE=$PWD source env.sh # source this from your run script too -mkdir build || true # make sure not to commit this to svn or git +mkdir -p build cd build -cmake .. # not required when re-compiling +cmake .. -DCMAKE_INSTALL_PREFIX=$HERE ${CMAKE_ARGS} make install -j`nproc --all` diff --git a/lib/parms/CMakeLists.txt b/lib/parms/CMakeLists.txt index 62ba062fa..71b2ee558 100644 --- a/lib/parms/CMakeLists.txt +++ b/lib/parms/CMakeLists.txt @@ -1,15 +1,24 @@ cmake_minimum_required(VERSION 3.4) -project(parms C) +project(fesom_parms C) + +# Set location to look for find_package modules +set( CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../cmake ) + +# Set -fPIC flag etc. +set( CMAKE_POSITION_INDEPENDENT_CODE ON ) # get our source files set(src_home ${CMAKE_CURRENT_LIST_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt file(GLOB all_sources ${src_home}/src/*.c ${src_home}/src/DDPQ/*.c) -include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/FindBLAS.cmake") +find_package(BLAS) +if(NOT TARGET MPI::MPI_C) + find_package(MPI REQUIRED COMPONENTS C) +endif() # create our library (set its name to name of this project) -add_library(${PROJECT_NAME} SHARED ${all_sources}) +add_library(${PROJECT_NAME} ${all_sources}) target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS USE_MPI REAL=double DBL FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 HAS_BLAS) @@ -18,8 +27,8 @@ target_include_directories(${PROJECT_NAME} PUBLIC $ PRIVATE ${src_home}/src/include ) - target_link_libraries(${PROJECT_NAME} PRIVATE ${BLAS_C_LIBRARIES} $ENV{UBUNTU_BLAS_LIBRARY}) +target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_C) if(${CMAKE_C_COMPILER_ID} STREQUAL "Intel") target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise) @@ -27,6 +36,3 @@ if(${CMAKE_C_COMPILER_ID} STREQUAL "Intel") target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) endif() endif() - -target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) -install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") diff --git a/lib/parms/src/parms_pc_schurras.c b/lib/parms/src/parms_pc_schurras.c index 979acdfe9..505dfc1c2 100755 --- a/lib/parms/src/parms_pc_schurras.c +++ b/lib/parms/src/parms_pc_schurras.c @@ -10,6 +10,11 @@ #endif #endif +/* Forward declarations */ +int parms_OperatorGetU(parms_Operator self, void **mat); +int parms_MatGetOffDiag(parms_Mat self, void **mat); +int parms_CommGetOdvlist(parms_Comm self, int **odvlist); + typedef struct schurras_data { parms_Operator op_out,op_in; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b37a7ed75..bb1118f1e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,30 @@ cmake_minimum_required(VERSION 3.9) -project(fesom C Fortran) +project(fesom LANGUAGES C CXX Fortran VERSION 2.5) + +message("[${PROJECT_NAME}]") + +### Setup project + +set(BUILD_SHARED_LIBS ON CACHE BOOL "Default to using shared libs") +set(CMAKE_LINK_DEPENDS_NO_SHARED ON) # relink of downstream libraries not required when shared library is rebuilt + +# Default build type: Release +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") + +# Set location to look for find_package modules +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake ${CMAKE_MODULE_PATH}) + +# Set Fortran module directory +set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/module/${PROJECT_NAME} ) + +# Set -fPIC flag etc. +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# Build-dir destinations +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) if(DEFINED ENV{FESOM_PLATFORM_STRATEGY}) set(FESOM_PLATFORM_STRATEGY $ENV{FESOM_PLATFORM_STRATEGY} CACHE STRING "switch to platform specific compile settings, this is usually determined via the env.sh script") @@ -44,47 +68,77 @@ elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo ) else() option(DISABLE_MULTITHREADING "disable asynchronous operations" OFF) endif() +message(STATUS "DISABLE_MULTITHREADING: ${DISABLE_MULTITHREADING}") + option(ENABLE_OPENACC "compile with OpenACC support" OFF) +message(STATUS "ENABLE_OPENACC: ${ENABLE_OPENACC}") set(NV_GPU_ARCH "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") option(ENABLE_OPENMP "build FESOM with OpenMP" OFF) +message(STATUS "ENABLE_OPENMP: ${ENABLE_OPENMP}") if(${ENABLE_OPENMP}) find_package(OpenMP REQUIRED) endif() -# option to trigger building a library version of FESOM -# we do not always build the library along with the executable to avoid having two targets here in the CMakeLists.txt -# two targets would allow e.g. setting different compiler options or preprocessor definition, which would be error prone. -option(BUILD_FESOM_AS_LIBRARY "Build a library instead of an executable" OFF) +option(USE_ICEPACK "Use ICEPACK" OFF) +message(STATUS "USE_ICEPACK: ${USE_ICEPACK}") + +find_package( NETCDF REQUIRED COMPONENTS C Fortran ) +find_package( MPI REQUIRED COMPONENTS C Fortran ) + + +option(ENABLE_IFS_INTERFACE "Enable IFS interface" OFF) +if(DEFINED FESOM_ENABLE_IFS_INTERFACE) + set(ENABLE_IFS_INTERFACE ${FESOM_ENABLE_IFS_INTERFACE}) # To distinguish option in a nested cmake project (bundle) +endif() +message(STATUS "ENABLE_IFS_INTERFACE: ${ENABLE_IFS_INTERFACE}") +if(ENABLE_IFS_INTERFACE) + if(USE_ICEPACK) + message(FATAL_ERROR "Could not enable IFS interface as it is incompatible with USE_ICEPACK") + endif() +endif() -set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/../.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") +option(ENABLE_MULTIO "Enable MultIO" OFF) +if(DEFINED FESOM_ENABLE_MULTIO) + set(ENABLE_MULTIO ${FESOM_ENABLE_MULTIO}) # To distinguish option in a nested cmake project (bundle) +endif() +message(STATUS "ENABLE_MULTIO: ${ENABLE_MULTIO}") +if(ENABLE_MULTIO) + find_package(multio) + if(NOT multio_FOUND) + message(FATAL_ERROR "Could not enable multio as multio could not be found") + endif() + if(NOT TARGET multio-fapi) + message(FATAL_ERROR "Could not enable multio as multio was not compiled with 'multio-fapi' target") + endif() +endif() # get our source files -set(src_home ${CMAKE_CURRENT_LIST_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt +set(src_home ${CMAKE_CURRENT_SOURCE_DIR}) # path to src directory starting from the dir containing our CMakeLists.txt if(${USE_ICEPACK}) file(GLOB sources_Fortran ${src_home}/*.F90 ${src_home}/icepack_drivers/*.F90 ${src_home}/icepack_drivers/Icepack/columnphysics/*.F90) -elseif(${BUILD_FESOM_AS_LIBRARY}) +elseif(ENABLE_IFS_INTERFACE) file(GLOB sources_Fortran ${src_home}/*.F90 ${src_home}/ifs_interface/*.F90) # ICEPACK + LIBRARY NOT SUPPORTED (YET) else() file(GLOB sources_Fortran ${src_home}/*.F90) endif() -#list(REMOVE_ITEM sources_Fortran ${src_home}/fesom_partition_init.F90) file(GLOB sources_C ${src_home}/*.c) -list(REMOVE_ITEM sources_C ${src_home}/psolve_feom.c) # does the file still exist? # generate a custom file from fesom_version_info.F90 which includes the current git SHA set(FESOM_ORIGINAL_VERSION_FILE ${src_home}/fesom_version_info.F90) set(FESOM_GENERATED_VERSION_FILE ${CMAKE_CURRENT_BINARY_DIR}/fesom_version_info-generated.F90) list(REMOVE_ITEM sources_Fortran ${FESOM_ORIGINAL_VERSION_FILE}) # we want to compile the generated file instead list(APPEND sources_Fortran ${FESOM_GENERATED_VERSION_FILE}) -add_custom_command(OUTPUT 5303B6F4_E4F4_45B2_A6E5_8E2B9FB5CDC4 ${FESOM_GENERATED_VERSION_FILE} # the first arg to OUTPUT is a name for a file we never create to make sure this command will run on every re-build (let our file be the second arg, as the first file is inadvertently removed by make) - COMMAND ${CMAKE_COMMAND} -DFESOM_ORIGINAL_VERSION_FILE=${FESOM_ORIGINAL_VERSION_FILE} -DFESOM_GENERATED_VERSION_FILE=${FESOM_GENERATED_VERSION_FILE} -P GitRepositoryInfo.cmake - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMENT "determining ${PROJECT_NAME} git SHA ...") +set_source_files_properties(${FESOM_GENERATED_VERSION_FILE} PROPERTIES GENERATED TRUE) + +add_custom_target(fesom_version_info-generated.F90 ALL + COMMENT "Determining ${PROJECT_NAME} git SHA ..." + COMMAND ${CMAKE_COMMAND} -DFESOM_ORIGINAL_VERSION_FILE=${FESOM_ORIGINAL_VERSION_FILE} -DFESOM_GENERATED_VERSION_FILE=${FESOM_GENERATED_VERSION_FILE} -P GitRepositoryInfo.cmake + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) #if(${FESOM_STANDALONE}) # list(REMOVE_ITEM sources_Fortran ${src_home}/cpl_driver.F90) @@ -96,36 +150,48 @@ list(REMOVE_ITEM sources_Fortran ${src_home}/fesom_main.F90) # depends on the metis library #add_subdirectory(../lib/metis-5.1.0 ${PROJECT_BINARY_DIR}/metis) #include_directories(../lib/metis-5.1.0/include) -# depends on the parms library +# depends on the fesom_parms library add_subdirectory(../lib/parms ${PROJECT_BINARY_DIR}/parms) add_subdirectory(async_threads_cpp) -include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindNETCDF.cmake) - -if(${BUILD_FESOM_AS_LIBRARY}) - add_library(${PROJECT_NAME}_C ${sources_C}) -else() add_library(${PROJECT_NAME}_C ${sources_C}) -endif() target_compile_definitions(${PROJECT_NAME}_C PRIVATE PARMS USE_MPI REAL=double DBL HAS_BLAS FORTRAN_UNDERSCORE VOID_POINTER_SIZE_8 SGI LINUX UNDER_ MPI2) -target_link_libraries(${PROJECT_NAME}_C parms) #metis +target_link_libraries(${PROJECT_NAME}_C PRIVATE fesom_parms) #metis +target_link_libraries(${PROJECT_NAME}_C PRIVATE MPI::MPI_C) -# create our binary or library (set its name to name of this project) -# we do not always build the library along with the executable to avoid having two targets here in the CMakeLists.txt -# two targets would allow e.g. setting different compiler options or preprocessor definition, which would be error prone. -if(${BUILD_FESOM_AS_LIBRARY}) - add_library(${PROJECT_NAME} ${sources_Fortran}) -else() - add_executable(${PROJECT_NAME} ${sources_Fortran} ${src_home}/fesom_main.F90) +# fesom library +add_library(${PROJECT_NAME} ${sources_Fortran}) +add_dependencies(${PROJECT_NAME} fesom_version_info-generated.F90) +target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243) +if(ENABLE_MULTIO) + target_compile_definitions(${PROJECT_NAME} PRIVATE __MULTIO) endif() -target_compile_definitions(${PROJECT_NAME} PRIVATE PARMS -DMETIS_VERSION=5 -DPART_WEIGHTED -DMETISRANDOMSEED=35243 __MULTIO) + +target_include_directories(${PROJECT_NAME} PRIVATE ${NETCDF_Fortran_INCLUDE_DIRECTORIES} ${OASIS_Fortran_INCLUDE_DIRECTORIES}) +target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTORIES} ${MPEU_Fortran_INCLUDE_DIRECTORIES}) +target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) +target_link_libraries(${PROJECT_NAME} PRIVATE fesom_parms) #metis +target_link_libraries(${PROJECT_NAME} PRIVATE fesom_async_threads) +target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran) +target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) +target_link_libraries(${PROJECT_NAME} PRIVATE ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) +if(ENABLE_OPENMP AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) + target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran) +endif() +if(ENABLE_MULTIO) + target_link_libraries(${PROJECT_NAME} PRIVATE multio-fapi) +endif() + + + if(${DISABLE_MULTITHREADING}) target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_MULTITHREADING) endif() if(${FESOM_COUPLED}) - include(${CMAKE_CURRENT_LIST_DIR}/../cmake/FindOASIS.cmake) + find_package(OASIS REQUIRED) target_compile_definitions(${PROJECT_NAME} PRIVATE __oasis) endif() if(${OIFS_COUPLED}) @@ -133,8 +199,7 @@ if(${OIFS_COUPLED}) endif() if(${USE_ICEPACK}) target_compile_definitions(${PROJECT_NAME} PRIVATE __icepack) -endif() -if(${BUILD_FESOM_AS_LIBRARY}) +elseif(${ENABLE_IFS_INTERFACE}) target_compile_definitions(${PROJECT_NAME} PRIVATE __ifsinterface) endif() if(${VERBOSE}) @@ -143,13 +208,10 @@ endif() if(${OPENMP_REPRODUCIBLE}) target_compile_definitions(${PROJECT_NAME} PRIVATE __openmp_reproducible) endif() + # CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel) if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel ) - if(${BUILD_FESOM_AS_LIBRARY}) - target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin -fpe0) # add -fpe0 for RAPS environment - else() - target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin) - endif() + target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin -fpe0) if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo) @@ -168,10 +230,11 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU ) target_compile_options(${PROJECT_NAME} PRIVATE -fallow-argument-mismatch) # gfortran v10 is strict about erroneous API calls: "Rank mismatch between actual argument at (1) and actual argument at (2) (scalar and rank-1)" endif() elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray ) + target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023) if(${ENABLE_OPENMP}) - target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -homp) + target_compile_options(${PROJECT_NAME} PRIVATE -homp) else() - target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -hnoomp) + target_compile_options(${PROJECT_NAME} PRIVATE -hnoomp) endif() elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) target_compile_definitions(${PROJECT_NAME} PRIVATE ENABLE_NVHPC_WORKAROUNDS) @@ -187,29 +250,57 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) endif() endif() -if(${BUILD_FESOM_AS_LIBRARY}) - target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) - target_compile_options(${PROJECT_NAME}_C PRIVATE -fPIC) + +# fesom.x executable +add_executable(${PROJECT_NAME}.x ${src_home}/fesom_main.F90) +target_link_libraries(${PROJECT_NAME}.x PUBLIC ${PROJECT_NAME}) + +### Export and installation + +# Set install directories +include(GNUInstallDirs) +if( NOT INSTALL_BIN_DIR ) + set( INSTALL_BIN_DIR ${CMAKE_INSTALL_BINDIR} ) endif() -target_include_directories(${PROJECT_NAME} PRIVATE ${NETCDF_Fortran_INCLUDE_DIRECTORIES} ${OASIS_Fortran_INCLUDE_DIRECTORIES}) -target_include_directories(${PROJECT_NAME} PRIVATE ${MCT_Fortran_INCLUDE_DIRECTORIES} ${MPEU_Fortran_INCLUDE_DIRECTORIES}) -target_include_directories(${PROJECT_NAME} PRIVATE ${SCRIP_Fortran_INCLUDE_DIRECTORIES}) -target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${OASIS_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_C ${MCT_Fortran_LIBRARIES} ${MPEU_Fortran_LIBRARIES} ${SCRIP_Fortran_LIBRARIES}) -target_link_libraries(${PROJECT_NAME} async_threads_cpp multio-fapi) -set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) -if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) - target_compile_options(${PROJECT_NAME} PRIVATE ${OpenMP_Fortran_FLAGS}) # currently we only have OpenMP in the Fortran part - target_link_libraries(${PROJECT_NAME} OpenMP::OpenMP_Fortran) +if( NOT INSTALL_LIB_DIR ) + set( INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR} ) endif() - -#set(FESOM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}/.." CACHE FILEPATH "directory where FESOM will be installed to via 'make install'") -if(${BUILD_FESOM_AS_LIBRARY}) - install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") - install(TARGETS ${PROJECT_NAME}_C EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") -else() - set(FESOM_INSTALL_FILEPATH "${FESOM_INSTALL_PREFIX}/bin/fesom.x") - get_filename_component(FESOM_INSTALL_PATH ${FESOM_INSTALL_FILEPATH} DIRECTORY) - get_filename_component(FESOM_INSTALL_NAME ${FESOM_INSTALL_FILEPATH} NAME) - install(PROGRAMS ${PROJECT_BINARY_DIR}/${PROJECT_NAME} DESTINATION ${FESOM_INSTALL_PATH} RENAME ${FESOM_INSTALL_NAME}) +if( NOT INSTALL_INCLUDE_DIR ) + set( INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR} ) endif() +set( INSTALL_CMAKE_DIR ${INSTALL_LIB_DIR}/cmake/${PROJECT_NAME} ) + +mark_as_advanced( INSTALL_BIN_DIR ) +mark_as_advanced( INSTALL_LIB_DIR ) +mark_as_advanced( INSTALL_INCLUDE_DIR ) +mark_as_advanced( INSTALL_CMAKE_DIR ) + +set( PROJECT_TARGETS_FILE "${PROJECT_BINARY_DIR}/${PROJECT_NAME}-targets.cmake" ) +file( REMOVE ${PROJECT_TARGETS_FILE} ) + + +foreach( tgt ${PROJECT_NAME}_C ${PROJECT_NAME} fesom_parms fesom_async_threads fesom.x ) + install( TARGETS ${tgt} + EXPORT ${PROJECT_NAME}-targets + RUNTIME DESTINATION ${INSTALL_BIN_DIR} + LIBRARY DESTINATION ${INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${INSTALL_LIB_DIR} ) + set_target_properties( ${tgt} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib ) + export( TARGETS ${tgt} APPEND FILE "${PROJECT_TARGETS_FILE}" ) +endforeach() + +include(CMakePackageConfigHelpers) +write_basic_package_version_file( + "${PROJECT_NAME}-config-version.cmake" + VERSION ${${PROJECT_NAME}_VERSION} + COMPATIBILITY AnyNewerVersion) + +configure_package_config_file(fesom-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/fesom-config.cmake + INSTALL_DESTINATION ${INSTALL_CMAKE_DIR}) + +install(EXPORT ${PROJECT_NAME}-targets DESTINATION "${INSTALL_CMAKE_DIR}") +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake + DESTINATION "${INSTALL_CMAKE_DIR}") diff --git a/src/GitRepositoryInfo.cmake b/src/GitRepositoryInfo.cmake index e5eb37c26..4f6a8172e 100644 --- a/src/GitRepositoryInfo.cmake +++ b/src/GitRepositoryInfo.cmake @@ -9,4 +9,6 @@ else() message("git not found, setting FESOM_GIT_SHA to: ${FESOM_GIT_SHA}") endif() -configure_file(${FESOM_ORIGINAL_VERSION_FILE} ${FESOM_GENERATED_VERSION_FILE} @ONLY) +configure_file(${FESOM_ORIGINAL_VERSION_FILE} ${FESOM_GENERATED_VERSION_FILE}.tmp @ONLY) +execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FESOM_GENERATED_VERSION_FILE}.tmp ${FESOM_GENERATED_VERSION_FILE}) +execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${FESOM_GENERATED_VERSION_FILE}.tmp) diff --git a/src/async_threads_cpp/CMakeLists.txt b/src/async_threads_cpp/CMakeLists.txt index f54186d8f..4c5e7b328 100644 --- a/src/async_threads_cpp/CMakeLists.txt +++ b/src/async_threads_cpp/CMakeLists.txt @@ -1,24 +1,18 @@ cmake_minimum_required(VERSION 3.4) -project(async_threads_cpp CXX C Fortran) # the FortranCInterface requires the C language to be enabled +project(fesom_async_threads CXX C Fortran) # the FortranCInterface requires the C language to be enabled + +set (CMAKE_CXX_STANDARD 11) # get our source files -file(GLOB sources_CXX ${CMAKE_CURRENT_LIST_DIR}/*.cpp) +file(GLOB sources ${CMAKE_CURRENT_LIST_DIR}/*.cpp) include(FortranCInterface) FortranCInterface_HEADER(ThreadsManagerFCMacros.h MACRO_NAMESPACE "ThreadsManagerFCMacros_" SYMBOLS init_ccall begin_ccall end_ccall) -add_library(${PROJECT_NAME} SHARED ${sources_CXX}) - +add_library(${PROJECT_NAME} ${sources} async_threads_module.F90) target_include_directories(${PROJECT_NAME} - PRIVATE ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_BINARY_DIR} + PRIVATE $ + PRIVATE $ + PUBLIC $ ) - -if(${CMAKE_CXX_COMPILER_ID} STREQUAL Cray ) - target_compile_options(${PROJECT_NAME} PRIVATE -hstd=c++11) -else() - target_compile_options(${PROJECT_NAME} PRIVATE -std=c++11) -endif() - -target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) -install(TARGETS ${PROJECT_NAME} EXPORT ifs_sp-targets DESTINATION "${FESOM_INSTALL_PREFIX}/lib") diff --git a/src/async_threads_module.F90 b/src/async_threads_cpp/async_threads_module.F90 similarity index 100% rename from src/async_threads_module.F90 rename to src/async_threads_cpp/async_threads_module.F90 diff --git a/src/fesom-config.cmake.in b/src/fesom-config.cmake.in new file mode 100644 index 000000000..dd5cab722 --- /dev/null +++ b/src/fesom-config.cmake.in @@ -0,0 +1,9 @@ +# This file will get included upon `find_package(fesom ... )` + +@PACKAGE_INIT@ + +include(${CMAKE_CURRENT_LIST_DIR}/fesom-targets.cmake) + +include(CMakeFindDependencyMacro) +# Add dependencies here of CMake projects fesom depends on that define targets. +#find_dependency(MPI COMPONENTS Fortran) \ No newline at end of file diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index a16c78a9a..4c43040bc 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -88,7 +88,9 @@ module fesom_module subroutine fesom_init(fesom_total_nsteps) use fesom_main_storage_module +#if defined(__MULTIO) use iom +#endif integer, intent(out) :: fesom_total_nsteps ! EO parameters logical mpi_is_initialized diff --git a/src/ifs_interface/iom.F90 b/src/ifs_interface/iom.F90 index 57783098e..806720760 100644 --- a/src/ifs_interface/iom.F90 +++ b/src/ifs_interface/iom.F90 @@ -6,13 +6,16 @@ !----------------------------------------------------- MODULE iom +#if defined(__MULTIO) USE multio_api +#endif USE, INTRINSIC :: iso_fortran_env, only: real64 IMPLICIT NONE PRIVATE - +#if defined(__MULTIO) TYPE(multio_handle) :: mio_handle +#endif INTEGER(8), PRIVATE :: mio_parent_comm PUBLIC iom_initialize, iom_init_server, iom_finalize @@ -43,6 +46,7 @@ SUBROUTINE multio_custom_error_handler(context, err) IMPLICIT NONE INTEGER(8), INTENT(INOUT) :: context ! Use mpi communicator as context INTEGER, INTENT(IN) :: err +#if defined(__MULTIO) INTEGER :: mpierr IF (err /= MULTIO_SUCCESS) THEN @@ -52,6 +56,9 @@ SUBROUTINE multio_custom_error_handler(context, err) context = MPI_UNDEFINED ENDIF ENDIF +#else + CALL ctl_stop('multio_custom_error_handler: FESOM compiled without MULTIO') +#endif END SUBROUTINE @@ -63,6 +70,7 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) INTEGER,INTENT(IN), OPTIONAL :: local_comm INTEGER,INTENT(OUT), OPTIONAL :: return_comm INTEGER,INTENT(IN), OPTIONAL :: global_comm +#if defined(__MULTIO) TYPE(multio_configuration) :: conf_ctx INTEGER :: err CHARACTER(len=16) :: err_str @@ -143,12 +151,15 @@ SUBROUTINE iom_initialize(client_id, local_comm, return_comm, global_comm ) IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('mio_handle%open_connections failed: ', multio_error_string(err)) END IF +#else + CALL ctl_stop("iom_initialize: FESOM built without multio") +#endif END SUBROUTINE iom_initialize SUBROUTINE iom_finalize() IMPLICIT NONE INTEGER :: err - +#if defined(__MULTIO) err = mio_handle%close_connections(); IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('mio_handle%close_connections failed: ', multio_error_string(err)) @@ -158,11 +169,15 @@ SUBROUTINE iom_finalize() IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('mio_handle%delete failed: ', multio_error_string(err)) END IF +#else + CALL ctl_stop("iom_finalize: FESOM built without multio") +#endif END SUBROUTINE iom_finalize SUBROUTINE iom_init_server(server_comm) IMPLICIT NONE INTEGER, INTENT(IN) :: server_comm +#if defined(__MULTIO) type(multio_configuration) :: conf_ctx INTEGER :: err CHARACTER(len=16) :: err_str @@ -225,6 +240,9 @@ SUBROUTINE iom_init_server(server_comm) IF (err /= MULTIO_SUCCESS) THEN CALL ctl_stop('conf_ctx%delete() failed: ', multio_error_string(err)) END IF +#else + CALL ctl_stop("iom_init_server: FESOM built without multio") +#endif END SUBROUTINE iom_init_server SUBROUTINE iom_send_fesom_domains(partit, mesh) @@ -233,11 +251,12 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) IMPLICIT NONE + TYPE(t_partit), INTENT(IN), TARGET :: partit + TYPE(t_mesh), intent(in), TARGET :: mesh +#if defined(__MULTIO) TYPE(multio_metadata) :: md INTEGER :: cerr INTEGER :: elem, elnodes(3), aux - TYPE(t_partit), INTENT(IN), TARGET :: partit - TYPE(t_mesh), intent(in), TARGET :: mesh INTEGER, DIMENSION(:), POINTER :: temp #include "../associate_part_def.h" @@ -326,12 +345,16 @@ SUBROUTINE iom_send_fesom_domains(partit, mesh) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_domains: egrid, md%delete() failed: ', multio_error_string(cerr)) END IF +#else + CALL ctl_stop("iom_send_fesom_domains: FESOM built without multio") +#endif END SUBROUTINE iom_send_fesom_domains SUBROUTINE iom_send_fesom_data(data) IMPLICIT NONE TYPE(iom_field_request), INTENT(INOUT) :: data +#if defined(__MULTIO) INTEGER :: cerr TYPE(multio_metadata) :: md @@ -389,6 +412,9 @@ SUBROUTINE iom_send_fesom_data(data) IF (cerr /= MULTIO_SUCCESS) THEN CALL ctl_stop('send_fesom_data: md%delete failed: ', multio_error_string(cerr)) END IF +#else + CALL ctl_stop("iom_send_fesom_data: FESOM built without multio") +#endif END SUBROUTINE SUBROUTINE ctl_stop(m1, m2, m3, m4) diff --git a/test/ifs_interface/check_exist.sh b/test/ifs_interface/check_exist.sh index e89a674e1..7d80ec346 100755 --- a/test/ifs_interface/check_exist.sh +++ b/test/ifs_interface/check_exist.sh @@ -2,11 +2,21 @@ set -e -FILE=./lib/libfesom.a -if [ -f "$FILE" ]; then +LIB_NAMES=("libfesom.a" "libfesom.so" "libfesom.dylib") + +libfesom_exists=false +for LIB_NAME in ${LIB_NAMES[@]}; do + FILE=./lib/${LIB_NAME} + if [ -f "$FILE" ]; then echo "$FILE compiled and linked." + libfesom_exists=true + fi +done + +if ${libfesom_exists}; then exit 0 else - echo "$FILE does not exist." + echo "libfesom not found" exit 1 fi + diff --git a/test/ifs_interface/configure.sh b/test/ifs_interface/configure.sh new file mode 100755 index 000000000..15de80700 --- /dev/null +++ b/test/ifs_interface/configure.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# custom build script in use at ECMWF + +set -e + +#cd ../../ +source env.sh ubuntu # source this from your run script too + +HERE=$PWD +echo "+ mkdir -p build" +mkdir -p build +echo "+ cd build" +cd build +echo + cmake -DCMAKE_INSTALL_PREFIX=$HERE -DBUILD_SHARED_LIBS=ON -DENABLE_IFS_INTERFACE=ON -DENABLE_MULTIO=OFF ${CMAKE_ARGS} .. # not required when re-compiling +cmake -DCMAKE_INSTALL_PREFIX=$HERE -DBUILD_SHARED_LIBS=ON -DENABLE_IFS_INTERFACE=ON -DENABLE_MULTIO=OFF ${CMAKE_ARGS} .. # not required when re-compiling +echo "+ make install -j`nproc --all`" +make install -j`nproc --all` + diff --git a/test/ifs_interface/configure_lib.sh b/test/ifs_interface/configure_lib.sh deleted file mode 100755 index cf1609bcb..000000000 --- a/test/ifs_interface/configure_lib.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# custom build script in use at ECMWF - -set -e - -LIB=no -while getopts "l" OPT -do - case "$OPT" in - l) LIB=yes;; - esac -done -shift $((OPTIND-1)) - -#cd ../../ -source env.sh ubuntu # source this from your run script too - -if [[ ${LIB} = yes ]]; then - mkdir build.lib || true # build dir for library - cd build.lib - cmake -DBUILD_FESOM_AS_LIBRARY=ON .. # not required when re-compiling -else - mkdir build || true # build dir for binary - cd build - cmake .. # not required when re-compiling -fi -make install -j`nproc --all` diff --git a/test_downstream/.gitignore b/test_downstream/.gitignore new file mode 100644 index 000000000..378eac25d --- /dev/null +++ b/test_downstream/.gitignore @@ -0,0 +1 @@ +build diff --git a/test_downstream/CMakeLists.txt b/test_downstream/CMakeLists.txt new file mode 100644 index 000000000..5425c63e9 --- /dev/null +++ b/test_downstream/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) + +project(test_downstream LANGUAGES Fortran) + +find_package(fesom REQUIRED) + +message("fesom found: ${fesom_DIR}") +message("fesom_VERSION: ${fesom_VERSION}") + +if(NOT TARGET fesom) + message(FATAL_ERROR "fesom was installed wrongly: fesom target not found") +endif() + +get_target_property(target_type fesom TYPE) +if( NOT target_type MATCHES "STATIC_LIBRARY|SHARED_LIBRARY") + message(FATAL_ERROR "fesom target's TYPE is expected to be STATIC_LIBRARY or SHARED_LIBRARY. Actual TYPE: ${target_type}") +endif () + diff --git a/test_downstream/README b/test_downstream/README new file mode 100644 index 000000000..c4baf5961 --- /dev/null +++ b/test_downstream/README @@ -0,0 +1,7 @@ +This dummy project can be used to test if fesom was installed and can be found correctly in downstream CMake projects + +Example use: + + export fesom_ROOT= # can be install or build dir!!! + cmake -S . -B build +