diff --git a/.github/tools/install-nvhpc.sh b/.github/tools/install-nvhpc.sh index ab95783e..2f90b681 100755 --- a/.github/tools/install-nvhpc.sh +++ b/.github/tools/install-nvhpc.sh @@ -62,14 +62,26 @@ ver="$(echo $version | tr -d . )" URL=$(curl -s "https://developer.nvidia.com/nvidia-hpc-sdk-$ver-downloads" | grep -oP "https://developer.download.nvidia.com/hpc-sdk/([0-9]{2}\.[0-9]+)/nvhpc_([0-9]{4})_([0-9]+)_Linux_$(uname -m)_cuda_([0-9\.]+).tar.gz" | sort | tail -1) FOLDER="$(basename "$(echo "${URL}" | grep -oP '[^/]+$')" .tar.gz)" -if [ ! -d "${TEMPORARY_FILES}/${FOLDER}" ]; then +MD5_HASH=$(curl -s "https://developer.download.nvidia.com/hpc-sdk/$ver/md5sum.txt" | grep -P "nvhpc_([0-9]{4})_([0-9]+)_Linux_$(uname -m)_cuda_([0-9\.]+).tar.gz" | awk '{print $1}') + +DOWNLOAD_REQUIRED=true +if [ -d "${TEMPORARY_FILES}/${FOLDER}" ]; then + echo "Download already present in ${TEMPORARY_FILES}/${FOLDER}" + if [ "${MD5_HASH}" = "$(cat ${TEMPORARY_FILES}/${FOLDER}/md5sum.txt)" ]; then + echo "MD5 hash matches expected value, using existing download" + DOWNLOAD_REQUIRED=false + else + echo "MD5 hash does not match expected value" + fi +fi + +if [ ${DOWNLOAD_REQUIRED} = true ]; then echo "Downloading ${TEMPORARY_FILES}/${FOLDER} from URL [${URL}]" mkdir -p ${TEMPORARY_FILES} curl --location \ --user-agent "pgi-travis (https://github.com/nemequ/pgi-travis)" \ "${URL}" | tar zx -C "${TEMPORARY_FILES}" -else - echo "Download already present in ${TEMPORARY_FILES}/${FOLDER}" + echo "${MD5_HASH}" >> "${TEMPORARY_FILES}/${FOLDER}/md5sum.txt" fi echo "+ ${TEMPORARY_FILES}/${FOLDER}/install" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2303f003..7b27a7d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,6 +30,8 @@ jobs: matrix: build_type: [Release,Debug] + lib_type: [Shared,Static] + feature: [None,DR_HOOK_PAPI,DR_HOOK_NVTX,DR_HOOK_ROCTX] name: - linux gnu-10 - linux gnu-12 @@ -117,6 +119,17 @@ jobs: coverage: false cmake_options: -DMPI_SLOTS=4 -DMPI_ARGS=--oversubscribe + - feature: DR_HOOK_PAPI + ctest_options: -LE PAPI_BROKEN_GITHUB_CI + + exclude: + - name: macos + feature: DR_HOOK_ROCTX + - name: macos + feature: DR_HOOK_NVTX + - name: macos + feature: DR_HOOK_PAPI + runs-on: ${{ matrix.os }} steps: - name: Checkout Repository @@ -129,11 +142,25 @@ jobs: echo "CXX=${{ matrix.compiler_cxx }}" >> $GITHUB_ENV echo "FC=${{ matrix.compiler_fc }}" >> $GITHUB_ENV + # https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=24.04&target_type=deb_network + if [[ "${{ matrix.feature }}" == "DR_HOOK_NVTX" ]]; then + wget https://developer.download.nvidia.com/compute/cuda/repos/${{ case(matrix.os == 'ubuntu-22.04', 'ubuntu2204', 'ubuntu2404') }}/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + echo "/usr/lib/nsight-systems/bin" >> "$GITHUB_PATH" + fi + + # https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/quick-start.html + if [[ "${{ matrix.feature }}" == "DR_HOOK_ROCTX" ]]; then + wget https://repo.radeon.com/amdgpu-install/7.2/ubuntu/${{ case(matrix.os == 'ubuntu-22.04', 'jammy', 'noble') }}/amdgpu-install_7.2.70200-1_all.deb + sudo apt install ./amdgpu-install_7.2.70200-1_all.deb + echo "/opt/rocm/bin" >> "$GITHUB_PATH" + fi + if [[ "${{ matrix.os }}" =~ macos ]]; then brew install ninja else sudo apt-get update - sudo apt-get install ninja-build + sudo apt-get install ninja-build ${{ case( matrix.feature == 'DR_HOOK_PAPI', 'libpapi-dev', matrix.feature == 'DR_HOOK_ROCTX', 'rocprofiler-sdk-roctx rocprofiler-sdk comgr hsa-amd-aqlprofile', matrix.feature == 'DR_HOOK_NVTX' && matrix.compiler != 'nvhpc-25.1' , 'nsight-systems cuda-nvtx-12-5 cuda-nvcc-12-5', '') }} fi printenv @@ -168,9 +195,13 @@ jobs: - name: Install NVHPC compiler if: contains( matrix.compiler, 'nvhpc' ) + env: + COMPILER_NAME: ${{ matrix.name }} shell: bash -eux {0} + # This uses a parameter expansion to strip the leading everything up to & including + # the last '-' run: | - ${FIAT_TOOLS}/install-nvhpc.sh --prefix /opt/nvhpc --version 25.1 + ${FIAT_TOOLS}/install-nvhpc.sh --prefix /opt/nvhpc --tmpdir ${{ env.DEPS_DIR }} --version ${COMPILER_NAME##*-} source /opt/nvhpc/env.sh echo "${NVHPC_DIR}/compilers/bin" >> $GITHUB_PATH echo "NVHPC_ROOT=${NVHPC_DIR}" >> $GITHUB_ENV @@ -212,7 +243,7 @@ jobs: - name: Build & Test id: build-test - uses: ecmwf-actions/build-package@v2 + uses: ecmwf/build-package@v2 with: self_coverage: ${{ matrix.coverage }} force_build: true @@ -226,8 +257,8 @@ jobs: dependency_cmake_options: | ecmwf/eckit: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF -DENABLE_ECKIT_CMD=OFF -DENABLE_ECKIT_SQL=OFF -DENABLE_MPI=ON -DENABLE_OMP=OFF" ecmwf/fckit: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF" - cmake_options: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ${{ matrix.cmake_options }} -DENABLE_MPI=ON" - ctest_options: "${{ matrix.ctest_options }}" + cmake_options: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DBUILD_SHARED_LIBS=${{ case(matrix.lib_type == 'Shared', 'ON', matrix.lib_type == 'Static', 'OFF', 'ON') }} ${{ case(matrix.feature == 'None', '', format('-DENABLE_{0}=ON', matrix.feature)) }} ${{ matrix.cmake_options }} -DENABLE_MPI=ON" + ctest_options: ${{ matrix.ctest_options }} # - name: Verify tools # run: | diff --git a/AUTHORS b/AUTHORS index 75af5d38..da316e93 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,6 +1,7 @@ Authors and Contributors ======================== +- A. Beggs (ECMWF) - W. Deconinck (ECMWF) - D. Dent (ECMWF) - R. El Khatib (Meteo France) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b69e007..07e417cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,6 +120,8 @@ if( target_build_type STREQUAL STATIC_LIBRARY OR TARGET fiat-static ) set( fiat_REQUIRES_PRIVATE_DEPENDENCIES TRUE ) endif() +set(FIAT_CMAKEMACROS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + ecbuild_install_project( NAME fiat ) ecbuild_print_summary() diff --git a/cmake/FindROCTX.cmake b/cmake/FindROCTX.cmake index d560ab3e..50e8bb9e 100644 --- a/cmake/FindROCTX.cmake +++ b/cmake/FindROCTX.cmake @@ -6,22 +6,23 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -set( HAVE_ROCPROFILER_SDK_ROCTX 0) +set( HAVE_ROCPROFILER_SDK_ROCTX 0) set( ROCTX_REQUIRED_VARS ROCTX_LIBRARIES ) -find_package( rocprofiler-sdk-roctx CONFIG PATHS ${ROCM_PATH}/lib ) -if( NOT rocprofiler-sdk-roctx_FOUND ) - ecbuild_info( "rocprofiler-sdk-roctx libraries not found" ) - if ( NOT DEFINED ROCM_PATH OR NOT ROCM_PATH_FOUND ) - find_path( +if ( NOT DEFINED ROCM_PATH OR NOT ROCM_PATH_FOUND ) + find_path( ROCM_PATH - NAMES include/roctracer/roctx.h + NAMES include/roctracer/roctx.h include/rocprofiler-sdk-roctx/roctx.h HINTS ENV ROCM_DIR ENV ROCM_PATH ENV HIP_PATH ENV ROCM_ROOT_DIR /opt/rocm - ) - ecbuild_info( "ROCM path: ${ROCM_PATH}" ) - endif() + ) + ecbuild_info( "ROCM path: ${ROCM_PATH}" ) +endif() - find_path( ROCTX_INCLUDE_DIRS NAMES roctx.h HINTS ${ROCM_PATH}/include/roctracer/ ) +find_package( rocprofiler-sdk-roctx CONFIG PATHS ${ROCM_PATH}/lib ${ROCM_PATH}/lib/cmake ) +if( NOT rocprofiler-sdk-roctx_FOUND ) + ecbuild_info( "rocprofiler-sdk-roctx libraries not found" ) + + find_path( ROCTX_INCLUDE_DIRS NAMES roctx.h HINTS ${ROCM_PATH}/include/roctracer/ ${ROCM_PATH}/include/rocprofiler-sdk-roctx ) list( APPEND ROCTX_REQUIRED_VARS ROCTX_INCLUDE_DIRS ) find_path( ROCTX_LIBRARY_PATH NAMES libroctx64.so HINTS ${ROCM_PATH}/lib/ ) @@ -31,8 +32,8 @@ if( NOT rocprofiler-sdk-roctx_FOUND ) else() if( TARGET ${rocprofiler-sdk-roctx_LIBRARIES} ) set( ROCTX_LIBRARIES ${rocprofiler-sdk-roctx_LIBRARIES} ) - set( ROCTX_INCLUDE_DIRS ${rocprofiler-sdk-roctx_INCLUDE_DIR} ) - list( APPEND ROCTX_REQUIRED_VARS ROCTX_INCLUDE_DIRS ) + set( ROCTX_INCLUDE_DIRS ${rocprofiler-sdk-roctx_INCLUDE_DIR} ) + list( APPEND ROCTX_REQUIRED_VARS ROCTX_INCLUDE_DIRS ) set( HAVE_ROCPROFILER_SDK_ROCTX 1 ) endif() endif() diff --git a/cmake/fiat-import.cmake.in b/cmake/fiat-import.cmake.in index c1c083fe..8e8de28b 100644 --- a/cmake/fiat-import.cmake.in +++ b/cmake/fiat-import.cmake.in @@ -15,6 +15,9 @@ set( fiat_VERSION_STR @fiat_VERSION_STR@ ) set( fiat_HAVE_MPI @fiat_HAVE_MPI@ ) set( fiat_HAVE_OMP @fiat_HAVE_OMP@ ) set( fiat_HAVE_FCKIT @fiat_HAVE_FCKIT@ ) +set( fiat_HAVE_DR_HOOK_NVTX @fiat_HAVE_DR_HOOK_NVTX@ ) +set( fiat_HAVE_DR_HOOK_ROCTX @fiat_HAVE_DR_HOOK_ROCTX@ ) +set( fiat_HAVE_DR_HOOK_PAPI @fiat_HAVE_DR_HOOK_PAPI@ ) set( fiat_HAVE_SINGLE_PRECISION @fiat_HAVE_SINGLE_PRECISION@ ) set( fiat_HAVE_DOUBLE_PRECISION @fiat_HAVE_DOUBLE_PRECISION@ ) set( fiat_HAVE_MPL_F08 @fiat_HAVE_MPL_F08@ ) @@ -22,6 +25,12 @@ set( fiat_REQUIRES_PRIVATE_DEPENDENCIES @fiat_REQUIRES_PRIVATE_DEPENDENCIES@ ) set( fiat_SOURCE_FILENAMES @fiat_SOURCE_FILENAMES@ ) + +# This variable picks up the source dir location for builds +set(FIAT_CMAKEMACROS_DIR @FIAT_CMAKEMACROS_DIR@) +list(APPEND CMAKE_MODULE_PATH ${FIAT_CMAKEMACROS_DIR}) + + if( fiat_HAVE_OMP AND NOT TARGET OpenMP::OpenMP_Fortran ) if( NOT CMAKE_Fortran_COMPILER_LOADED ) enable_language( Fortran ) @@ -39,6 +48,18 @@ if( fiat_HAVE_MPI AND NOT TARGET MPI::MPI_Fortran ) endif() endif() +if( fiat_HAVE_DR_HOOK_NVTX ) + find_dependency( NVTX ) +endif() + +if( fiat_HAVE_DR_HOOK_ROCTX ) + find_dependency( ROCTX ) +endif() + +if( fiat_HAVE_DR_HOOK_PAPI ) + find_dependency( PAPI ) +endif() + if( fiat_HAVE_FCKIT AND NOT TARGET fckit ) find_dependency( fckit HINTS ${CMAKE_CURRENT_LIST_DIR}/../fckit @fckit_DIR@ ) endif() diff --git a/src/fiat/CMakeLists.txt b/src/fiat/CMakeLists.txt index af8c2d3e..a8bf320b 100644 --- a/src/fiat/CMakeLists.txt +++ b/src/fiat/CMakeLists.txt @@ -148,6 +148,7 @@ foreach( LIB_TYPE ${LIB_TYPES} ) target_link_libraries (${fiatlib} PRIVATE ${NVTX_LIBRARIES}) target_include_directories(${fiatlib} PRIVATE ${NVTX_INCLUDE_DIRS}) endif() + set(LINK_FIAT_MPI_LIBRARIES PUBLIC_LIBS ${FIAT_MPI_LIBRARIES}) if (HAVE_DR_HOOK_ROCTX) # Files from within DrHook diff --git a/src/fiat/drhook/drhook.c b/src/fiat/drhook/drhook.c index 8078c5d0..92ab17db 100644 --- a/src/fiat/drhook/drhook.c +++ b/src/fiat/drhook/drhook.c @@ -2659,8 +2659,11 @@ process_options() char* opt_name = p; /* Assume that everything is OFF by default */ if (strequ(p,"ALL")) { /* all except profiler data */ - opt_papi = opt_gethwm = opt_getstk = opt_getrss = opt_getpag = opt_walltime = opt_cputime = opt_cycles = 1; + opt_gethwm = opt_getstk = opt_getrss = opt_getpag = opt_walltime = opt_cputime = opt_cycles = 1; opt_calls = 1; +#ifdef DR_HOOK_HAVE_PAPI + opt_papi = 1; +#endif any_memstat++; } else if (strequ(p,"MEM") || strequ(p,"MEMORY")) { @@ -2732,7 +2735,9 @@ process_options() opt_cpuprof = 0; /* Note: Switches cpuprof OFF */ opt_calls = 1; opt_cycles = 1; +#ifdef DR_HOOK_HAVE_PAPI opt_papi = 1; +#endif } else if (strequ(p,"CPUPROF")) { opt_cpuprof = 1; @@ -2824,7 +2829,7 @@ process_options() #if defined(DR_HOOK_HAVE_PAPI) if (opt_papi) { - newline = 0; + int newline = 0; env = getenv("DR_HOOK_PAPI_COUNTERS"); if (env) { const char delim[] = ", \t/"; diff --git a/src/fiat/drhook/extensions/papi/drhook_papi.c b/src/fiat/drhook/extensions/papi/drhook_papi.c index 3ba5fb35..d8d34bd1 100644 --- a/src/fiat/drhook/extensions/papi/drhook_papi.c +++ b/src/fiat/drhook/extensions/papi/drhook_papi.c @@ -75,14 +75,14 @@ void drhook_papi_print(char* s, long_long* a, int header){ PAPI_event_code_to_name(papi_counter_event_codes[i], event_name); sprintf(&msg[strlen(msg)]," %16s",event_name); } - printf("%s\n",msg); + fprintf(stderr, "%s\n",msg); } sprintf(msg,"%s",s); for (int i=0;i 0) { snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, library version mismatch between compilation and run!\n"); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } if (paperr == PAPI_EINVAL){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, PAPI_EINVAL\n"); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } if (paperr == PAPI_ENOMEM){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, PAPI_ENOMEM\n"); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } if (paperr == PAPI_ESBSTR){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, PAPI_ESBSTR\n"); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } if (paperr == PAPI_ESYS){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, PAPI_ESYS\n"); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } else { snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, unknown error code: %d\n", paperr); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } } @@ -224,7 +224,7 @@ int drhook_papi_init(int rank){ if( paperr != PAPI_OK ) { snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, thread init failed (%s)",PAPI_strerror(paperr)); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } @@ -235,7 +235,7 @@ int drhook_papi_init(int rank){ nthreads); if (drhook_papi_rank==0 && !silent) { - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); } drhook_papi_event_set=malloc_drhook(nthreads*sizeof(int)); @@ -252,7 +252,7 @@ int drhook_papi_init(int rank){ drhook_papi_state=drhook_papi_running; if (drhook_papi_rank==0 && !silent) { - printf("DRHOOK:PAPI: Initialisation sucess\n"); + fprintf(stderr, "DRHOOK:PAPI: Initialisation sucess\n"); } return 1; } @@ -266,27 +266,27 @@ int drhook_papi_start_threads(int* events){ papiErr=PAPI_create_eventset(&events[thread]); if (papiErr != PAPI_OK){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, create event set failed (%s) \n",PAPI_strerror(papiErr)); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } - if (!silent) printf("DRHOOK:PAPI: Event set %d created for thread %d\n",events[thread],thread); + if (!silent) fprintf(stderr, "DRHOOK:PAPI: Event set %d created for thread %d\n",events[thread],thread); if (!silent && drhook_papi_rank==0 && thread==0) - printf("DRHOOK:PAPI: Attempting to add events to event set:\n"); + fprintf(stderr, "DRHOOK:PAPI: Attempting to add events to event set:\n"); for (int counter=0; counter < drhook_papi_num_counters(); counter ++) { int eventCode; if (!silent && drhook_papi_rank==0 && thread==0) { snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: %s", papi_counter_names[counter]); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); } papiErr=PAPI_event_name_to_code(papi_counter_names[counter], &eventCode); if (papiErr != PAPI_OK){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, event name to code failed for %s (%s)", papi_counter_names[counter], PAPI_strerror(papiErr)); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); PAPI_perror("initPapi"); return 0; } @@ -295,33 +295,33 @@ int drhook_papi_start_threads(int* events){ papiErr=PAPI_add_event(events[thread],eventCode); if (papiErr!=PAPI_OK){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error, add_event failed: %d (%s)",papiErr,PAPI_strerror(papiErr)); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); if (papiErr == PAPI_EINVAL) { - printf("Invalid argument. "); + fprintf(stderr, "Invalid argument. "); } else if (papiErr == PAPI_ENOMEM) { - printf("Out of memory. "); + fprintf(stderr, "Out of memory. "); } else if (papiErr == PAPI_ENOEVST) { - printf("EventSet does not exist. "); + fprintf(stderr, "EventSet does not exist. "); } else if (papiErr == PAPI_EISRUN) { - printf("EventSet is running. "); + fprintf(stderr, "EventSet is running. "); } else if (papiErr == PAPI_ECNFLCT) { - printf("Conflict. "); + fprintf(stderr, "Conflict. "); } else if (papiErr == PAPI_ENOEVNT) { - printf("Preset not available. "); + fprintf(stderr, "Preset not available. "); } - printf("This is an error within PAPI and not DrHook. DrHook is only reporting the error it received.\n"); + fprintf(stderr, "This is an error within PAPI and not DrHook. DrHook is only reporting the error it received.\n"); return 0; } else { #if defined(DEBUG) snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Added code=%d to Event set %d",eventCode, events[thread]); if (thread==0) { - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); } #endif } @@ -332,31 +332,31 @@ int drhook_papi_start_threads(int* events){ papiErr = PAPI_list_events(events[thread], checkEvents, &number); if (papiErr != PAPI_OK){ snprintf(pmsg,STD_MSG_LEN,"DRHOOK:PAPI: Error querying events - %d=%s",papiErr,PAPI_strerror(papiErr)); - printf("%s\n",pmsg); + fprintf(stderr, "%s\n",pmsg); return 0; } #if defined(DEBUG) for (int counter=0;counter