cuda.yml

name: cuda

on:
  push:
    branches-ignore:
      - 'dependabot/**'
  pull_request:

concurrency:
  group: ${{ github.ref }}-${{ github.head_ref }}-cuda
  cancel-in-progress: true

jobs:
  check_changes:
    uses: ./.github/workflows/check_changes.yml

  # Build libamrex and all tests with CUDA 12.2
  tests-cuda12-2:
    name: CUDA@12.2 Release Single Precision Particle [tests]
    runs-on: ubuntu-22.04
    needs: check_changes
    if: needs.check_changes.outputs.has_non_docs_changes == 'true'
    steps:
    - uses: actions/checkout@v7
    - name: Dependencies
      run: |
        .github/workflows/dependencies/ubuntu_free_disk_space.sh
        .github/workflows/dependencies/dependencies_nvcc.sh 12.2
        .github/workflows/dependencies/dependencies_ccache.sh
    - name: Set Up Cache
      uses: actions/cache@v6
      with:
        path: ~/.cache/ccache
        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
        restore-keys: |
             ccache-${{ github.workflow }}-${{ github.job }}-git-
    - name: Build & Install
      env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches"}
      run: |
        export CCACHE_COMPRESS=1
        export CCACHE_COMPRESSLEVEL=10
        export CCACHE_MAXSIZE=235M
        ccache -z

        export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
        export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
        which nvcc || echo "nvcc not in PATH!"

        cmake -S . -B build                              \
            -DCMAKE_CXX_STANDARD=20                      \
            -DCMAKE_VERBOSE_MAKEFILE=ON                  \
            -DAMReX_PRECISION=DOUBLE                     \
            -DAMReX_PARTICLES_PRECISION=SINGLE           \
            -DAMReX_FFT=ON                               \
            -DAMReX_EB=ON                                \
            -DAMReX_ENABLE_TESTS=ON                      \
            -DAMReX_FORTRAN=OFF                          \
            -DAMReX_GPU_BACKEND=CUDA                     \
            -DCMAKE_C_COMPILER=$(which gcc)              \
            -DCMAKE_CXX_COMPILER=$(which g++)            \
            -DCMAKE_CUDA_HOST_COMPILER=$(which g++)      \
            -DCMAKE_Fortran_COMPILER=$(which gfortran)   \
            -DCMAKE_CUDA_ARCHITECTURES=80                \
            -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
            -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache        \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache

        cmake --build build -j 4

        ccache -s
        du -hs ~/.cache/ccache

  # Build libamrex and all tests with CUDA 12.6
  tests-cuda12:
    name: CUDA@12.6 Release [tests]
    runs-on: ubuntu-24.04
    needs: check_changes
    if: needs.check_changes.outputs.has_non_docs_changes == 'true'
    steps:
    - uses: actions/checkout@v7
    - name: Dependencies
      run: |
        .github/workflows/dependencies/ubuntu_free_disk_space.sh
        .github/workflows/dependencies/dependencies_nvcc.sh 12.6
        .github/workflows/dependencies/dependencies_ccache.sh
    - name: Set Up Cache
      uses: actions/cache@v6
      with:
        path: ~/.cache/ccache
        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
        restore-keys: |
             ccache-${{ github.workflow }}-${{ github.job }}-git-
    - name: Build & Install
      env: {CXXFLAGS: "-fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wlogical-op -Wmisleading-indentation -Wduplicated-cond -Wduplicated-branches"}
      run: |
        export CCACHE_COMPRESS=1
        export CCACHE_COMPRESSLEVEL=10
        export CCACHE_MAXSIZE=235M
        ccache -z

        export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
        export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
        which nvcc || echo "nvcc not in PATH!"

        cmake -S . -B build                              \
            -DCMAKE_CXX_STANDARD=20                      \
            -DCMAKE_VERBOSE_MAKEFILE=ON                  \
            -DAMReX_FFT=ON                               \
            -DAMReX_EB=ON                                \
            -DAMReX_ENABLE_TESTS=ON                      \
            -DAMReX_FORTRAN=OFF                          \
            -DAMReX_GPU_BACKEND=CUDA                     \
            -DCMAKE_C_COMPILER=$(which gcc)              \
            -DCMAKE_CXX_COMPILER=$(which g++)            \
            -DCMAKE_CUDA_HOST_COMPILER=$(which g++)      \
            -DCMAKE_Fortran_COMPILER=$(which gfortran)   \
            -DCMAKE_CUDA_ARCHITECTURES=70                \
            -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
            -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache        \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache

        cmake --build build -j 4

        ccache -s
        du -hs ~/.cache/ccache

  # Build libamrex and all tests for CUDA with LLVM Clang + CTK 12.5
  tests-cuda-clang:
    name: Clang@19 CUDA@12.5 Release [tests]
    runs-on: ubuntu-24.04
    needs: check_changes
    if: needs.check_changes.outputs.has_non_docs_changes == 'true'
    env:
      CC: clang-19
      CXX: clang++-19
      CUDACXX: clang++-19
      CUDAHOSTCXX: clang++-19
      CUDAARCHS: "80"
    steps:
    - uses: actions/checkout@v7
    - name: Dependencies
      run: |
        .github/workflows/dependencies/ubuntu_free_disk_space.sh
        .github/workflows/dependencies/dependencies_clang.sh 19
        .github/workflows/dependencies/dependencies_nvcc_2404.sh 12.5
        .github/workflows/dependencies/dependencies_ccache.sh
    - name: Set Up Cache
      uses: actions/cache@v6
      with:
        path: ~/.cache/ccache
        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
        restore-keys: |
             ccache-${{ github.workflow }}-${{ github.job }}-git-
    - name: Build & Install
      run: |
        export CCACHE_COMPRESS=1
        export CCACHE_COMPRESSLEVEL=10
        export CCACHE_MAXSIZE=60M
        export CCACHE_DEPEND=1
        ccache -z

        export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
        export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}

        cmake -S . -B build                              \
            -DCMAKE_CXX_STANDARD=20                      \
            -DCMAKE_VERBOSE_MAKEFILE=ON                  \
            -DAMReX_MPI=OFF                              \
            -DAMReX_FFT=ON                               \
            -DAMReX_EB=ON                                \
            -DAMReX_ENABLE_TESTS=ON                      \
            -DAMReX_FORTRAN=OFF                          \
            -DAMReX_GPU_BACKEND=CUDA                     \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache        \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache

        cmake --build build -j 4

        ccache -s
        du -hs ~/.cache/ccache

  # Build libamrex and all tests with NVHPC (recent supported)
  tests-nvhpc-nvcc:
    name: NVHPC NVCC/NVC++ Release [tests]
    runs-on: ubuntu-24.04
    needs: check_changes
    if: needs.check_changes.outputs.has_non_docs_changes == 'true'
    steps:
    - uses: actions/checkout@v7
    - name: Dependencies
      run: |
        .github/workflows/dependencies/ubuntu_free_disk_space.sh
        .github/workflows/dependencies/dependencies_nvhpc.sh
        .github/workflows/dependencies/dependencies_ccache.sh
    - name: Set Up Cache
      uses: actions/cache@v6
      with:
        path: ~/.cache/ccache
        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
        restore-keys: |
             ccache-${{ github.workflow }}-${{ github.job }}-git-
    - name: Build & Install
      env: {CXXFLAGS: "-Werror -Wall -Wextra -Wpedantic -Wshadow --diag_suppress=code_is_unreachable"}
      run: |
        export CCACHE_COMPRESS=1
        export CCACHE_COMPRESSLEVEL=10
        export CCACHE_MAXSIZE=300M
        ccache -z

        source /etc/profile.d/modules.sh
        nvhpc_version=`ls -v /opt/nvidia/hpc_sdk/modulefiles/nvhpc/ | tail -n 1`
        module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/${nvhpc_version}

        which nvcc || echo "nvcc not in PATH!"
        which nvc++ || echo "nvc++ not in PATH!"
        which nvc || echo "nvc not in PATH!"
        which nvfortran || echo "nvfortran not in PATH!"
        nvcc --version
        nvc++ --version
        nvc --version
        nvfortran --version
        cmake --version

        cmake -S . -B build                              \
            -DCMAKE_CXX_STANDARD=20                      \
            -DCMAKE_VERBOSE_MAKEFILE=ON                  \
            -DAMReX_ENABLE_TESTS=ON                      \
            -DAMReX_TEST_TYPE=Small                      \
            -DAMReX_EB=ON                                \
            -DAMReX_FFT=ON                               \
            -DAMReX_FORTRAN=ON                           \
            -DAMReX_FORTRAN_INTERFACES=ON                \
            -DAMReX_GPU_BACKEND=CUDA                     \
            -DCMAKE_C_COMPILER=$(which nvc)              \
            -DCMAKE_CXX_COMPILER=$(which nvc++)          \
            -DCMAKE_CUDA_HOST_COMPILER=$(which nvc++)    \
            -DCMAKE_Fortran_COMPILER=$(which nvfortran)  \
            -DCMAKE_CUDA_ARCHITECTURES=80                \
            -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
            -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache        \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache

        cmake --build build -j 4

        ccache -s
        du -hs ~/.cache/ccache

  # Build 3D libamrex cuda build with configure
  configure-3d-cuda:
    name: CUDA@13.0 [configure 3D]
    runs-on: ubuntu-24.04
    needs: check_changes
    if: needs.check_changes.outputs.has_non_docs_changes == 'true'
    steps:
    - uses: actions/checkout@v7
    - name: Dependencies
      run: |
        .github/workflows/dependencies/ubuntu_free_disk_space.sh
        .github/workflows/dependencies/dependencies_nvcc_2404.sh 13.0
        .github/workflows/dependencies/dependencies_ccache.sh
    - name: Set Up Cache
      uses: actions/cache@v6
      with:
        path: ~/.cache/ccache
        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
        restore-keys: |
             ccache-${{ github.workflow }}-${{ github.job }}-git-
    - name: Build & Install
      run: |
        export CCACHE_COMPRESS=1
        export CCACHE_COMPRESSLEVEL=10
        export CCACHE_MAXSIZE=215M
        ccache -z

        export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
        ./configure --dim 3 --with-cuda yes --enable-eb yes --enable-xsdk-defaults yes --with-fortran no --enable-fft yes
        make -j4 CXXSTD=c++20 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS="-fno-operator-names -Werror=deprecated-declarations" CCACHE=ccache CUDA_ARCH="8.0 9.0"
        make install

        ccache -s
        du -hs ~/.cache/ccache