diff --git a/test_mom/CMakeLists.txt b/test_mom/CMakeLists.txt index 82cd7df..c9b23e6 100644 --- a/test_mom/CMakeLists.txt +++ b/test_mom/CMakeLists.txt @@ -17,8 +17,9 @@ cmake_minimum_required(VERSION 3.20) project(test_mom CXX) +enable_language(C Fortran) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -26,6 +27,23 @@ if(AMReX_GPU_BACKEND STREQUAL "CUDA") enable_language(CUDA) endif() +# --- MPI based tests ------------------------------------------------------ +option(MPI_UNIT_TESTS "Build and run MPI based unit tests." OFF) + +# --- GTest ---------------------------------------------------------------- +option(TEST_MOM_FETCH_GTEST + "Download GoogleTest via FetchContent if find_package fails" ON) + +# --- Captured-data directory (used by --data-dir CLI flag) --------------- +set(TEST_MOM_DATA_DIR "" + CACHE PATH "Directory containing the captured Fortran I/O .bin/.meta files") +if(NOT TEST_MOM_DATA_DIR) + message(FATAL_ERROR + "TEST_MOM_DATA_DIR is not set. Pass\n" + " -DTEST_MOM_DATA_DIR=/path/to/captured/fixtures\n" + "on the cmake configure line.") +endif() + # --- AMReX ---------------------------------------------------------------- # Prefer an installed AMReX if available; otherwise build the in-tree source. set(AMREX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../amrex" @@ -44,10 +62,6 @@ if(NOT AMReX_FOUND) ${CMAKE_CURRENT_BINARY_DIR}/_amrex) endif() -# --- GTest ---------------------------------------------------------------- -option(TEST_MOM_FETCH_GTEST - "Download GoogleTest via FetchContent if find_package fails" ON) - find_package(GTest QUIET) if(NOT GTest_FOUND) if(TEST_MOM_FETCH_GTEST) @@ -69,28 +83,10 @@ endif() enable_testing() include(GoogleTest) -# --- Captured-data directory (used by --data-dir CLI flag) --------------- -set(TEST_MOM_DATA_DIR "" - CACHE PATH "Directory containing the captured Fortran I/O .bin/.meta files") -if(NOT TEST_MOM_DATA_DIR) - message(FATAL_ERROR - "TEST_MOM_DATA_DIR is not set. Pass\n" - " -DTEST_MOM_DATA_DIR=/path/to/captured/fixtures\n" - "on the cmake configure line.") -endif() - # --- MOM C++ kernels library (target: mom_continuity) --------------------- add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../mom/cpp ${CMAKE_CURRENT_BINARY_DIR}/mom_cpp) -# --- mom_continuity_ppm test ---------------------------------------------- -add_executable(test_mom_continuity_ppm - test_mom_main.cpp - common/captured_io.cpp - common/amrex_assertions.cpp - mom_continuity_ppm/test_mom_continuity_ppm.cpp -) - # Every source in this target includes AMReX headers. With the CUDA backend # those headers use __host__/__device__ qualifiers and CUDA kernel-launch # syntax, so nvcc must compile them. CMake won't infer this from .cpp, so we @@ -101,23 +97,18 @@ if(AMReX_GPU_BACKEND STREQUAL "CUDA") common/captured_io.cpp common/amrex_assertions.cpp mom_continuity_ppm/test_mom_continuity_ppm.cpp + boundary/test_boundary_exchange.cpp PROPERTIES LANGUAGE CUDA) endif() -target_include_directories(test_mom_continuity_ppm - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/common -) -target_link_libraries(test_mom_continuity_ppm - PRIVATE - mom_continuity - GTest::gtest -) -gtest_discover_tests(test_mom_continuity_ppm - EXTRA_ARGS --data-dir=${TEST_MOM_DATA_DIR} - # Discover at test-run time rather than build time. With the CUDA backend - # the binary calls amrex::Initialize() which aborts on hosts without a GPU - # (e.g. Derecho login nodes), so running it at POST_BUILD would break the - # build. - DISCOVERY_MODE PRE_TEST -) +include(cmake/add_mpi_test.cmake) +add_subdirectory(common) + +add_library(data_driven_shared_main OBJECT test_mom_main.cpp) +target_link_libraries(data_driven_shared_main PUBLIC common) + +if(MPI_UNIT_TESTS) + add_subdirectory(boundary) +else() + add_subdirectory(mom_continuity_ppm) +endif() diff --git a/test_mom/boundary/CMakeLists.txt b/test_mom/boundary/CMakeLists.txt new file mode 100644 index 0000000..e64b160 --- /dev/null +++ b/test_mom/boundary/CMakeLists.txt @@ -0,0 +1 @@ +add_mpi_test(test_boundary_exchange test_boundary_exchange.cpp ${TEST_MOM_DATA_DIR}) diff --git a/test_mom/boundary/test_boundary_exchange.cpp b/test_mom/boundary/test_boundary_exchange.cpp new file mode 100644 index 0000000..00f5b89 --- /dev/null +++ b/test_mom/boundary/test_boundary_exchange.cpp @@ -0,0 +1,94 @@ +#include + +#include + +#include +#include +#include +#include + +#include "amrex_assertions.hpp" +#include "captured_io.hpp" +#include "data_dir.hpp" + +using test_mom::expect_arrays_equal; + +TEST(BoundaryExchange, BoundaryMatchesAfterOneTimeStep) { + int myRank = amrex::ParallelDescriptor::MyProc(); + test_mom::CapturedFile captured(test_mom::data_dir / + ("MOM_fixed_initialization_" + std::format("{:04}", myRank))); + + // Number of halo cells in x,y directions + int nihalo = captured.integer("G%Domain%nihalo"); + int njhalo = captured.integer("G%Domain%njhalo"); + // Total computational domain not including ghost cells + int niglobal = captured.integer("G%Domain%niglobal"); + int njglobal = captured.integer("G%Domain%njglobal"); + // Global location of current grid with ghost cells counting negative at origin + // using 0-based indexing + int isd_global = captured.integer("G%isd_global"); + int jsd_global = captured.integer("G%jsd_global"); + + auto bathyTBefore = captured.fab_host("G%bathyT_brefore", isd_global, jsd_global); + auto bathyTAfter = captured.fab_host("G%bathyT_after" , isd_global, jsd_global); + + // Sets up data domain region + int iLocalDataGridSize = bathyTBefore.box().length(0) - (2*nihalo); + int jLocalDataGridSize = bathyTBefore.box().length(1) - (2*njhalo); + + amrex::BoxList boxList; + amrex::Vector processBoxList; + + // Assumes equally sized grids + for(int i = 0; i < amrex::ParallelDescriptor::NProcs(); i++) + { + test_mom::CapturedFile rankc(test_mom::data_dir / + ("MOM_fixed_initialization_" + std::format("{:04}", i))); + + // Each rank needs to map the region every other rank convers. + int iGridStartLoc = rankc.integer("G%isd_global") + nihalo; + int jGridStartLoc = rankc.integer("G%jsd_global") + njhalo; + amrex::IntVect lo(iGridStartLoc, jGridStartLoc, 0); + amrex::IntVect hi(lo[0] + iLocalDataGridSize - 1, lo[1] + jLocalDataGridSize - 1, 0); + + boxList.push_back(amrex::Box(lo, hi)); + processBoxList.push_back(i); + } + + // Setup multifab + amrex::BoxArray boxes(boxList); + amrex::DistributionMapping dm(processBoxList); + amrex::IntVect fourRowTwo2DGhostCells{nihalo, njhalo, 0}; + amrex::MultiFab mf(boxes, dm, /*ncomp=*/1, fourRowTwo2DGhostCells); + + // Setup geometry + amrex::Box globalComputationalDomain(amrex::IntVect( 0, 0, 0), + amrex::IntVect(niglobal - 1, njglobal - 1, 0)); + amrex::RealBox realBox({0.0, 0.0, 0.0}, {1.0, 1.0, 1.0}); + amrex::Geometry geom(globalComputationalDomain, &realBox); + + // Verify that the loops iterate only the expected number of times + int mf_iterations = 0; + for (amrex::MFIter mfi(mf, /*tiling=*/false); mfi.isValid(); ++mfi) + { + amrex::FArrayBox& localValidFab = mf[mfi]; + ASSERT_EQ(bathyTBefore.box(), localValidFab.box()); + + localValidFab.copy(bathyTBefore); + mf_iterations++; + } + ASSERT_EQ(1, mf_iterations); + + mf.FillBoundary(geom.periodicity()); + + mf_iterations = 0; + for (amrex::MFIter mfi(mf, /*tiling=*/false); mfi.isValid(); ++mfi) + { + amrex::FArrayBox& localValidFab = mf[mfi]; + ASSERT_EQ(bathyTAfter.box(), localValidFab.box()); + + expect_arrays_equal(bathyTAfter, localValidFab, "G%bathyT"); + mf_iterations++; + } + ASSERT_EQ(1, mf_iterations); +} diff --git a/test_mom/cmake/add_mpi_test.cmake b/test_mom/cmake/add_mpi_test.cmake new file mode 100644 index 0000000..5f594a4 --- /dev/null +++ b/test_mom/cmake/add_mpi_test.cmake @@ -0,0 +1,27 @@ + +function(add_mpi_test TEST_TARGET_NAME TEST_FILE DATA_DIR) + + set(prefix TIM) + set(optionalValues "") + set(singleValues MAX_PES) + set(multiValues "") + + include(CMakeParseArguments) + cmake_parse_arguments(${prefix} "${optionalValues}" "${singleValues}" "${multiValues}" ${ARGN}) + + if(DEFINED TIM_MAX_PES) + if (TIM_MAX_PES NOT MATCHES "^[1-9][0-9]*$") + message(FATAL_ERROR "MAX_PES expected a valid positive integer; recieved ${TIM_MAX_PES}") + endif() + else() + set(TIM_MAX_PES 4) + endif() + + add_executable(${TEST_TARGET_NAME} ${TEST_FILE}) + target_sources(${TEST_TARGET_NAME} PRIVATE $) + target_link_libraries(${TEST_TARGET_NAME} PRIVATE common) + + add_test(NAME MPI_${TEST_TARGET_NAME} + COMMAND ${MPIEXEC_EXECUTABLE} --tag-output ${MPIEXEC_NUMPROC_FLAG} ${TIM_MAX_PES} ./${TEST_TARGET_NAME} --data-dir=${CMAKE_CURRENT_SOURCE_DIR}/${DATA_DIR}) + set_tests_properties(MPI_${TEST_TARGET_NAME} PROPERTIES PROCESSORS ${TIM_MAX_PES}) +endfunction() diff --git a/test_mom/common/CMakeLists.txt b/test_mom/common/CMakeLists.txt new file mode 100644 index 0000000..dde066c --- /dev/null +++ b/test_mom/common/CMakeLists.txt @@ -0,0 +1,8 @@ +set(COMMON_SRC + amrex_assertions.cpp + captured_io.cpp) + +add_library(common OBJECT ${COMMON_SRC}) + +target_link_libraries(common PUBLIC GTest::gtest AMReX::amrex) +target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/test_mom/common/captured_io.cpp b/test_mom/common/captured_io.cpp index 617e4fc..2198c5a 100644 --- a/test_mom/common/captured_io.cpp +++ b/test_mom/common/captured_io.cpp @@ -118,7 +118,11 @@ amrex::Box CapturedFile::box(const std::string& name) const { amrex::IntVect(idxE[0] - 1, idxE[1] - 1, idxE[2] - 1)); } -amrex::FArrayBox CapturedFile::fab_host(const std::string& name) const { +amrex::FArrayBox CapturedFile::fab_host(const std::string& name, + int startx, + int starty, + int startz) const +{ const auto& e = lookup(name, "RealArray_t"); std::size_t off = e.offset - 1; int ndim = read_be_i32(bin_, off); @@ -127,17 +131,11 @@ amrex::FArrayBox CapturedFile::fab_host(const std::string& name) const { "captured_io: fab expects 2D or 3D arrays (got ndim=" + std::to_string(ndim) + ") for '" + name + "'"); } - int shape[3] = {1, 1, 1}; + amrex::IntVect shape(1, 1, 1); for (int i = 0; i < ndim; ++i) shape[i] = read_be_i32(bin_, off); for (int i = 0; i < ndim; ++i) { int lb = read_be_i32(bin_, off); int ub = read_be_i32(bin_, off); - if (lb != 1) { - throw std::runtime_error( - "captured_io: fab assumes Fortran lb==1 (got " + - std::to_string(lb) + " on dim " + std::to_string(i) + - ") for '" + name + "'"); - } if (ub - lb + 1 != shape[i]) { throw std::runtime_error( "captured_io: fab bounds inconsistent with shape on dim " + @@ -154,15 +152,15 @@ amrex::FArrayBox CapturedFile::fab_host(const std::string& name) const { std::to_string(nelem) + ", expected " + std::to_string(expected) + ")"); } - amrex::Box sbx(amrex::IntVect(0, 0, 0), - amrex::IntVect(shape[0] - 1, shape[1] - 1, shape[2] - 1)); + amrex::IntVect start(startx, starty, startz); + amrex::IntVect end = start + (shape - 1); + amrex::Box sbx(start, end); amrex::FArrayBox fab(sbx, 1, amrex::The_Pinned_Arena()); auto arr = fab.array(); - // Fortran column-major in the file: i fastest, then j, then k. - for (int k = 0; k < shape[2]; ++k) - for (int j = 0; j < shape[1]; ++j) - for (int i = 0; i < shape[0]; ++i) - arr(i, j, k) = read_be_f64(bin_, off); + for(int k = sbx.smallEnd(2); k < sbx.bigEnd(2); ++k) + for(int j = sbx.smallEnd(1); j < sbx.bigEnd(1); ++j) + for(int i = sbx.smallEnd(0); i < sbx.bigEnd(0); ++i) + arr(i, j, k) = read_be_f64(bin_, off); return fab; } diff --git a/test_mom/common/captured_io.hpp b/test_mom/common/captured_io.hpp index 1061c40..55c3f2e 100644 --- a/test_mom/common/captured_io.hpp +++ b/test_mom/common/captured_io.hpp @@ -61,7 +61,7 @@ class CapturedFile { explicit CapturedFile(const std::filesystem::path& base); amrex::Box box (const std::string& name) const; - amrex::FArrayBox fab_host (const std::string& name) const; + amrex::FArrayBox fab_host (const std::string& name, int startx=0, int starty=0, int startz=0) const; amrex::FArrayBox fab_device(const std::string& name) const; double real64 (const std::string& name) const; bool logical (const std::string& name) const; diff --git a/test_mom/mom_continuity_ppm/CMakeLists.txt b/test_mom/mom_continuity_ppm/CMakeLists.txt new file mode 100644 index 0000000..3aafe70 --- /dev/null +++ b/test_mom/mom_continuity_ppm/CMakeLists.txt @@ -0,0 +1,11 @@ +add_executable(test_mom_continuity_ppm test_mom_continuity_ppm.cpp) +target_sources(test_mom_continuity_ppm PRIVATE $) +target_link_libraries(test_mom_continuity_ppm PRIVATE common mom_continuity) +gtest_discover_tests(test_mom_continuity_ppm + EXTRA_ARGS --data-dir=${TEST_MOM_DATA_DIR} + # Discover at test-run time rather than build time. With the CUDA backend + # the binary calls amrex::Initialize() which aborts on hosts without a GPU + # (e.g. Derecho login nodes), so running it at POST_BUILD would break the + # build. + DISCOVERY_MODE PRE_TEST +)