From 3f09d7db48456bfaebd9b0bb7ce8d122145c083c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 4 Sep 2026 10:32:51 -0700 Subject: [PATCH 1/4] CI: submit configure and build results to CDash Our CDash dashboard showed test results only. Every build reported `hasconfigure=false` and `hascompilation=false`, so configure and build timings, and all compiler warnings and errors, were missing. The cause is that CDash only receives those results when the steps run *through* CTest, which records them in `Configure.xml` and `Build.xml`. All three CI systems called `cmake` and `cmake --build` directly and then ran `ctest -D ExperimentalTest -D ExperimentalSubmit`, which produces a `Test.xml` and nothing else. Add `Tools/CI/ctest_dashboard.sh`, a wrapper around the CTest script `Tools/CI/ctest_dashboard.cmake`, that drives the configure and build via `ctest_configure()` and `ctest_build()`. Using a CTest script rather than `ctest -D ExperimentalConfigure` is deliberate: the latter needs a `DartConfiguration.tcl`, which only exists after a first `cmake` run, so it would time a warm re-configure and under-report by minutes. The test step stays with the caller, so CI keeps its existing `ctest` command line including options such as `--no-tests=error`, which `ctest_test()` does not offer. All steps share one dashboard tag, so a single `-D ExperimentalSubmit` uploads them as one CDash build. User-facing: - CDash now shows configure and build times per CI build, and every compiler warning and error with its source file and line. - The GitHub Actions compile matrix reports to CDash for the first time, under site `GitHub-Actions`. Internal: - The wrapper appends `-DBUILDNAME`/`-DSITE` to the configure options. Without them `include(CTest)` labels `Test.xml` with the host name and a generic build name, and CDash files it as a separate build from `Configure.xml` and `Build.xml`. - Build parallelism moves from `cmake --build -j N` to `CMAKE_BUILD_PARALLEL_LEVEL`, since the CTest build step has no `-j`. - Submission still only happens on merges to `development`. Build-only jobs submit from the script via `CDASH_SUBMIT=ON`; jobs that also test submit from a step that runs even when the build or tests failed, so a broken build reaches the dashboard. - Not converted: Windows (PowerShell/cmd), the GNU Make CUDA job (no CMake), and the clang-tidy, CodeQL and sanitizer workflows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CxAc564RY4pPkGbhio7fBm --- .azure-pipelines.yml | 34 +++++---- .github/workflows/cuda.yml | 23 ++++-- .github/workflows/hip.yml | 17 +++-- .github/workflows/insitu.yml | 26 ++++--- .github/workflows/intel.yml | 17 +++-- .github/workflows/macos.yml | 16 +++-- .github/workflows/ubuntu.yml | 33 ++++++--- .gitlab/ci.yaml | 77 +++++++++++---------- AGENTS.md | 11 +++ Docs/source/developers/how_to_test.rst | 50 ++++++++++++++ Tools/CI/ctest_dashboard.cmake | 96 ++++++++++++++++++++++++++ Tools/CI/ctest_dashboard.sh | 61 ++++++++++++++++ 12 files changed, 369 insertions(+), 92 deletions(-) create mode 100644 Tools/CI/ctest_dashboard.cmake create mode 100755 Tools/CI/ctest_dashboard.sh diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 39b0fa7a171..cbc55e26590 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -32,6 +32,7 @@ jobs: # FIXME remove unused variables variables: BLASPP_HOME: '/usr/local' + CDASH_SITE: 'Azure' CEI_SUDO: 'sudo' CEI_TMP: '/tmp/cei' CMAKE_GENERATOR: 'Ninja' @@ -184,17 +185,16 @@ jobs: set -o nounset errexit pipefail # display disk space usage df -h - # configure + # configure and build through CTest, so that CDash records their timings + # as well as compiler warnings and errors export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON" export WARPX_TEST_FLAGS="-DWarpX_TEST_CLEANUP=ON -DWarpX_TEST_FPETRAP=ON -DWarpX_BACKTRACE_INFO=ON" - cmake -S . -B build \ - -DBUILDNAME="${CDASH_BUILD_NAME}" \ - -DSITE=Azure \ + # CTest builds via `cmake --build`, which has no -j of its own + export CMAKE_BUILD_PARALLEL_LEVEL=2 + Tools/CI/ctest_dashboard.sh build \ ${AMReX_CMAKE_FLAGS} \ ${WARPX_CMAKE_FLAGS} \ ${WARPX_TEST_FLAGS} - # build - cmake --build build -j 2 displayName: 'Build' - bash: | @@ -210,17 +210,21 @@ jobs: # allow oversubscription for Open MPI 4 and 5 export OMPI_MCA_rmaps_base_oversubscribe=1 export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe - # determine if the build was triggered by a push to the development branch + # run tests (exclude pytest.AMReX when running Python tests) + ctest --test-dir build --output-on-failure -E AMReX -D ExperimentalTest + displayName: 'Test' + + # Upload Configure.xml, Build.xml and Test.xml as one CDash build. This runs + # even when the build or the tests failed, which is exactly what the dashboard + # should show. Only merges to `development` are submitted; PR pipelines write + # the XML but keep it. + - bash: | + set -o nounset errexit pipefail if [[ "$(Build.SourceBranch)" == "refs/heads/development" ]]; then - # run tests (exclude pytest.AMReX when running Python tests) - # and submit results to CDash as Experimental - ctest --test-dir build --output-on-failure -E AMReX \ - -D ExperimentalTest -D ExperimentalSubmit - else - # run tests (exclude pytest.AMReX when running Python tests) - ctest --test-dir build --output-on-failure -E AMReX + ctest --test-dir build -D ExperimentalSubmit fi - displayName: 'Test' + displayName: 'CDash Submit' + condition: always() - bash: | # set options diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 01f8d0190ce..26843582cf5 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -11,6 +11,13 @@ concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-cuda cancel-in-progress: true +# Configure and build through CTest, so that CDash records their timings as well +# as compiler warnings and errors. Only merges to `development` are submitted; +# PR runs produce the XML locally and drop it. +env: + CDASH_SITE: GitHub-Actions + CDASH_SUBMIT: ${{ github.event_name == 'push' && github.ref == 'refs/heads/development' && 'ON' || 'OFF' }} + jobs: check_changes: @@ -25,6 +32,8 @@ jobs: env: CXXFLAGS: "-Werror" CMAKE_GENERATOR: Ninja + CDASH_BUILD_NAME: Linux-NVCC-3D-SP + CMAKE_BUILD_PARALLEL_LEVEL: 3 steps: - name: Free More Disk Space uses: ax3l/free-disk-space@main @@ -83,7 +92,7 @@ jobs: export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" - cmake -S . -B build_sp \ + Tools/CI/ctest_dashboard.sh build_sp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_COMPUTE=CUDA \ -DWarpX_EB=ON \ @@ -95,7 +104,6 @@ jobs: -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON - cmake --build build_sp -j 3 python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools[core] wheel @@ -161,9 +169,11 @@ jobs: runs-on: ubuntu-24.04 needs: check_changes if: ${{ github.event.pull_request.draft == false && needs.check_changes.outputs.has_non_docs_changes == 'true' }} - #env: - # # For NVHPC, Ninja is slower than the default: - # CMAKE_GENERATOR: Ninja + env: + CDASH_BUILD_NAME: Linux-NVHPC-3D + CMAKE_BUILD_PARALLEL_LEVEL: 4 + # For NVHPC, Ninja is slower than the default: + #CMAKE_GENERATOR: Ninja steps: - name: Free More Disk Space uses: ax3l/free-disk-space@main @@ -208,7 +218,7 @@ jobs: export CUDACXX=$(which nvcc) export CUDAHOSTCXX=${CXX} - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_COMPUTE=CUDA \ -DWarpX_EB=ON \ @@ -218,7 +228,6 @@ jobs: -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON - cmake --build build -j 4 # work-around for mpi4py 3.1.1 build system issue with using # a GNU-built Python executable with non-GNU Python modules diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 29ada931b8a..8f990494037 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -11,6 +11,13 @@ concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-hip cancel-in-progress: true +# Configure and build through CTest, so that CDash records their timings as well +# as compiler warnings and errors. Only merges to `development` are submitted; +# PR runs produce the XML locally and drop it. +env: + CDASH_SITE: GitHub-Actions + CDASH_SUBMIT: ${{ github.event_name == 'push' && github.ref == 'refs/heads/development' && 'ON' || 'OFF' }} + jobs: check_changes: @@ -23,6 +30,8 @@ jobs: env: CXXFLAGS: "-Werror -Wno-deprecated-declarations -Wno-error=pass-failed" CMAKE_GENERATOR: Ninja + CDASH_BUILD_NAME: Linux-HIP-3D-SP + CMAKE_BUILD_PARALLEL_LEVEL: 4 needs: check_changes if: ${{ github.event.pull_request.draft == false && needs.check_changes.outputs.has_non_docs_changes == 'true' }} steps: @@ -52,7 +61,7 @@ jobs: export CXX=$(which clang++) export CC=$(which clang) - cmake -S . -B build_sp \ + Tools/CI/ctest_dashboard.sh build_sp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DAMReX_AMD_ARCH=gfx900 \ -DWarpX_COMPUTE=HIP \ @@ -62,7 +71,6 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ -DWarpX_FFT=ON - cmake --build build_sp -j 4 export WARPX_MPI=OFF export PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ @@ -78,6 +86,8 @@ jobs: env: CXXFLAGS: "-Werror -Wno-deprecated-declarations -Wno-error=pass-failed" CMAKE_GENERATOR: Ninja + CDASH_BUILD_NAME: Linux-HIP-2D-DP + CMAKE_BUILD_PARALLEL_LEVEL: 4 needs: check_changes if: ${{ github.event.pull_request.draft == false && needs.check_changes.outputs.has_non_docs_changes == 'true' }} steps: @@ -107,7 +117,7 @@ jobs: export CXX=$(which clang++) export CC=$(which clang) - cmake -S . -B build_2d \ + Tools/CI/ctest_dashboard.sh build_2d \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DAMReX_AMD_ARCH=gfx900 \ -DWarpX_DIMS=2 \ @@ -118,7 +128,6 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ -DWarpX_FFT=ON - cmake --build build_2d -j 4 export WARPX_MPI=OFF export PYWARPX_LIB_DIR=$PWD/build_2d/lib/site-packages/pywarpx/ diff --git a/.github/workflows/insitu.yml b/.github/workflows/insitu.yml index a725bd15f76..fe2ead7a02f 100644 --- a/.github/workflows/insitu.yml +++ b/.github/workflows/insitu.yml @@ -11,6 +11,13 @@ concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-insituvis cancel-in-progress: true +# Configure and build through CTest, so that CDash records their timings as well +# as compiler warnings and errors. Only merges to `development` are submitted; +# PR runs produce the XML locally and drop it. +env: + CDASH_SITE: GitHub-Actions + CDASH_SUBMIT: ${{ github.event_name == 'push' && github.ref == 'refs/heads/development' && 'ON' || 'OFF' }} + jobs: check_changes: @@ -27,20 +34,18 @@ jobs: CC: gcc CMAKE_PREFIX_PATH: /ascent/install/lib/cmake/ OMP_NUM_THREADS: 1 + CDASH_BUILD_NAME: Linux-GCC-Ascent + CMAKE_BUILD_PARALLEL_LEVEL: 4 container: image: alpinedav/ascent:0.9.3 steps: - uses: actions/checkout@v7 - - name: Configure + - name: Configure & Build run: | . /ascent_docker_setup_env.sh - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DWarpX_ASCENT=ON \ -DWarpX_COMPUTE=NOACC - - name: Build - run: | - . /ascent_docker_setup_env.sh - cmake --build build -j 4 - name: Test run: | cp Examples/Physics_applications/laser_acceleration/inputs_base_3d . @@ -70,6 +75,8 @@ jobs: CATALYST_DEBUG: 1 CATALYST_IMPLEMENTATION_PATHS: /opt/paraview/lib/catalyst OMP_NUM_THREADS: 1 + CDASH_BUILD_NAME: Linux-GCC-Catalyst-2D3D + CMAKE_BUILD_PARALLEL_LEVEL: 10 # Container build scripts: # https://gitlab.kitware.com/christos.tsolakis/catalyst-amrex-docker-images @@ -77,14 +84,11 @@ jobs: image: kitware/paraview:ci-catalyst-amrex-warpx-20240828 steps: - uses: actions/checkout@v7 - - name: Configure + - name: Configure & Build run: | - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DWarpX_DIMS="2;3" \ -DWarpX_CATALYST=ON - - name: Build - run: | - cmake --build build -j 10 - name: 2D Test run: | cp Examples/Tests/field_ionization/inputs_test_2d_ionization_lab . diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 4702dac1328..bc260c54de6 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -11,6 +11,13 @@ concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-intel cancel-in-progress: true +# Configure and build through CTest, so that CDash records their timings as well +# as compiler warnings and errors. Only merges to `development` are submitted; +# PR runs produce the XML locally and drop it. +env: + CDASH_SITE: GitHub-Actions + CDASH_SUBMIT: ${{ github.event_name == 'push' && github.ref == 'refs/heads/development' && 'ON' || 'OFF' }} + jobs: check_changes: @@ -25,6 +32,8 @@ jobs: # https://github.com/BLAST-WarpX/warpx/issues/3442 env: CXXFLAGS: "-Werror -Wno-deprecated -Wno-error=pass-failed -Wno-tautological-constant-compare" + CDASH_BUILD_NAME: Linux-ICX-3D-SP + CMAKE_BUILD_PARALLEL_LEVEL: 4 # For oneAPI, Ninja is slower than the default: # CMAKE_GENERATOR: Ninja needs: check_changes @@ -60,14 +69,13 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools[core] wheel - cmake -S . -B build_sp \ + Tools/CI/ctest_dashboard.sh build_sp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_EB=OFF \ -DWarpX_PYTHON=ON \ -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 4 cmake --build build_sp --target pip_install ccache -s @@ -89,6 +97,8 @@ jobs: # https://github.com/BLAST-WarpX/warpx/issues/3442 env: CXXFLAGS: "-Werror -Wno-deprecated -Wno-tautological-constant-compare" + CDASH_BUILD_NAME: Linux-SYCL-3D-MP + CMAKE_BUILD_PARALLEL_LEVEL: 4 # For oneAPI, Ninja is slower than the default: # CMAKE_GENERATOR: Ninja needs: check_changes @@ -122,7 +132,7 @@ jobs: export CC=$(which icx) export CXXFLAGS="-fsycl ${CXXFLAGS}" - cmake -S . -B build_dpsp \ + Tools/CI/ctest_dashboard.sh build_dpsp \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_COMPUTE=SYCL \ @@ -136,7 +146,6 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ -DWarpX_PARTICLE_PRECISION=SINGLE - cmake --build build_dpsp -j 4 ccache -s du -hs ~/.cache/ccache diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index d4c162e522c..a4d4ff36265 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -11,6 +11,13 @@ concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-macos cancel-in-progress: true +# Configure and build through CTest, so that CDash records their timings as well +# as compiler warnings and errors. Only merges to `development` are submitted; +# PR runs produce the XML locally and drop it. +env: + CDASH_SITE: GitHub-Actions + CDASH_SUBMIT: ${{ github.event_name == 'push' && github.ref == 'refs/heads/development' && 'ON' || 'OFF' }} + jobs: check_changes: @@ -24,6 +31,7 @@ jobs: if: ${{ github.event.pull_request.draft == false && needs.check_changes.outputs.has_non_docs_changes == 'true' }} env: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE + CMAKE_BUILD_PARALLEL_LEVEL: 3 # For macOS, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja steps: @@ -73,21 +81,21 @@ jobs: export CXXFLAGS="-Werror -Wno-error=pass-failed -Wno-error=c++23-attribute-extensions" - cmake -S . -B build_dp \ + CDASH_BUILD_NAME=macOS-AppleClang-3D-DP \ + Tools/CI/ctest_dashboard.sh build_dp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_EB=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF - cmake --build build_dp -j 3 - cmake -S . -B build_sp \ + CDASH_BUILD_NAME=macOS-AppleClang-3D-SP \ + Tools/CI/ctest_dashboard.sh build_sp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_EB=OFF \ -DWarpX_PYTHON=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 3 cmake --build build_sp --target pip_install du -hs ~/Library/Caches/ccache diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a3cef98f22f..9d1926735d4 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -11,6 +11,13 @@ concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-ubuntu cancel-in-progress: true +# Configure and build through CTest, so that CDash records their timings as well +# as compiler warnings and errors. Only merges to `development` are submitted; +# PR runs produce the XML locally and drop it. +env: + CDASH_SITE: GitHub-Actions + CDASH_SUBMIT: ${{ github.event_name == 'push' && github.ref == 'refs/heads/development' && 'ON' || 'OFF' }} + jobs: check_changes: @@ -24,6 +31,8 @@ jobs: if: ${{ github.event.pull_request.draft == false && needs.check_changes.outputs.has_non_docs_changes == 'true' }} env: CXXFLAGS: "-Werror" + CDASH_BUILD_NAME: Linux-GCC-Minimal-RZ3D + CMAKE_BUILD_PARALLEL_LEVEL: 4 steps: - uses: actions/checkout@v7 - name: install dependencies @@ -43,13 +52,12 @@ jobs: export CCACHE_MAXSIZE=100M ccache -z - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ -DWarpX_EB=OFF \ -DWarpX_MPI=OFF \ -DWarpX_QED=OFF - cmake --build build -j 4 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz @@ -65,6 +73,8 @@ jobs: CXXFLAGS: "-Werror" CXX: "g++-13" CC: "gcc-13" + CDASH_BUILD_NAME: Linux-GCC13-1D2D + CMAKE_BUILD_PARALLEL_LEVEL: 4 steps: - uses: actions/checkout@v7 - name: install dependencies @@ -84,7 +94,7 @@ jobs: export CCACHE_MAXSIZE=100M ccache -z - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="1;2" \ @@ -92,8 +102,6 @@ jobs: -DWarpX_FFT=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_QED_TOOLS=ON - - cmake --build build -j 4 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_base_1d ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d @@ -112,6 +120,8 @@ jobs: env: CXX: "g++-13" CC: "gcc-13" + CDASH_BUILD_NAME: Linux-GCC13-RZ3D-SP + CMAKE_BUILD_PARALLEL_LEVEL: 4 steps: - uses: actions/checkout@v7 - name: install dependencies @@ -135,7 +145,7 @@ jobs: # because the compilation of blaspp raises warnings. export CXXFLAGS="-Werror" - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ @@ -144,8 +154,6 @@ jobs: -DWarpX_PRECISION=SINGLE \ -DWarpX_PARTICLE_PRECISION=SINGLE \ -DWarpX_QED_TABLE_GEN=ON - - cmake --build build -j 4 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz @@ -160,6 +168,8 @@ jobs: env: CMAKE_GENERATOR: Ninja CXXFLAGS: "-Werror" + CDASH_BUILD_NAME: Linux-GCC-ABLASTR + CMAKE_BUILD_PARALLEL_LEVEL: 4 steps: - uses: actions/checkout@v7 - name: install dependencies @@ -180,11 +190,10 @@ jobs: export CCACHE_MAXSIZE=100M ccache -z - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ -DWarpX_LIB=OFF - cmake --build build -j 4 ccache -s du -hs ~/.cache/ccache @@ -197,6 +206,8 @@ jobs: env: CC: clang CXX: clang++ + CDASH_BUILD_NAME: Linux-Clang-pywarpx + CMAKE_BUILD_PARALLEL_LEVEL: 4 # On CI for this test, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja steps: @@ -223,7 +234,7 @@ jobs: export CXXFLAGS="-Werror -Wno-error=pass-failed" - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ diff --git a/.gitlab/ci.yaml b/.gitlab/ci.yaml index af1bbbb361e..87d740ff713 100644 --- a/.gitlab/ci.yaml +++ b/.gitlab/ci.yaml @@ -27,16 +27,31 @@ - python3 -m pip install --upgrade -r Regression/requirements.txt - python3 -m pip cache purge +# Upload Configure.xml, Build.xml and Test.xml as one CDash build. +# +# Used as `after_script`, so that it also runs when the configure, build or test +# step failed -- a broken build is exactly what the dashboard should show. Only +# merges to `development` are submitted; PR pipelines write the XML but keep it. +.cdash-submit: &cdash-submit + - | + if [[ "${CI_COMMIT_BRANCH}" == "development" ]]; then + ctest --test-dir build -D ExperimentalSubmit + fi + NVIDIA-H100: extends: .install-dependencies tags: [nvidia-h100] image: nvcr.io/nvidia/cuda:12.8.0-devel-ubuntu24.04 + variables: + CDASH_BUILD_NAME: GPU-H100-CUDA-SP + CDASH_SITE: Frank_UO + CMAKE_BUILD_PARALLEL_LEVEL: "16" script: + # configure and build through CTest, so that CDash records their timings + # as well as compiler warnings and errors - | - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DBUILDNAME=GPU-H100-CUDA-SP \ - -DSITE=Frank_UO \ -DWarpX_COMPUTE=CUDA \ -DWarpX_EB=ON \ -DWarpX_PYTHON=OFF \ @@ -46,24 +61,22 @@ NVIDIA-H100: -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON - - cmake --build build -j 16 - export OMPI_ALLOW_RUN_AS_ROOT=1 - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - export AMREX_DEFAULT_INIT="amrex.the_arena_init_size=0" - | - if [[ "${CI_COMMIT_BRANCH}" == "development" ]]; then - # run tests and submit results to CDash as Experimental - ctest --test-dir build -R test_3d_laser_acceleration_btd \ - --output-on-failure --no-tests=error -D ExperimentalTest -D ExperimentalSubmit - else - ctest --test-dir build -R test_3d_laser_acceleration_btd \ - --output-on-failure --no-tests=error - fi + ctest --test-dir build -R test_3d_laser_acceleration_btd \ + --output-on-failure --no-tests=error -D ExperimentalTest + after_script: *cdash-submit AMD-MI300: extends: .install-dependencies tags: [amd-mi300] image: rocm/dev-ubuntu-24.04:6.4.3-complete + variables: + CDASH_BUILD_NAME: GPU-MI300-HIP-SP + CDASH_SITE: Frank_UO + CMAKE_BUILD_PARALLEL_LEVEL: "16" script: - export ROCM_PATH=/opt/rocm-6.4.3 - export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:$PATH @@ -71,13 +84,13 @@ AMD-MI300: - export HIP_PATH=$ROCM_PATH - export HIP_CLANG_PATH=$ROCM_PATH/llvm/bin - export CMAKE_PREFIX_PATH=$ROCM_PATH:$CMAKE_PREFIX_PATH + # configure and build through CTest, so that CDash records their timings + # as well as compiler warnings and errors - | - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DCMAKE_C_COMPILER=$(which clang) \ -DCMAKE_CXX_COMPILER=$(which clang++) \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DBUILDNAME=GPU-MI300-HIP-SP \ - -DSITE=Frank_UO \ -DAMReX_AMD_ARCH=gfx942 \ -DWarpX_COMPUTE=HIP \ -DWarpX_EB=ON \ @@ -86,19 +99,13 @@ AMD-MI300: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ -DWarpX_FFT=ON - - cmake --build build -j 16 - export OMPI_ALLOW_RUN_AS_ROOT=1 - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - export AMREX_DEFAULT_INIT="amrex.the_arena_init_size=0" - | - if [[ "${CI_COMMIT_BRANCH}" == "development" ]]; then - # run tests and submit results to CDash as Experimental - ctest --test-dir build -R test_3d_laser_acceleration_btd \ - --output-on-failure --no-tests=error -D ExperimentalTest -D ExperimentalSubmit - else - ctest --test-dir build -R test_3d_laser_acceleration_btd \ - --output-on-failure --no-tests=error - fi + ctest --test-dir build -R test_3d_laser_acceleration_btd \ + --output-on-failure --no-tests=error -D ExperimentalTest + after_script: *cdash-submit Intel-PVC: # TODO: Re-enable the Intel PVC/SYCL runner. Runtime issues. @@ -107,14 +114,18 @@ Intel-PVC: extends: .install-dependencies tags: [intel-data-center-max-1100] image: intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04 + variables: + CDASH_BUILD_NAME: GPU-PVC-SYCL-MP + CDASH_SITE: Frank_UO + CMAKE_BUILD_PARALLEL_LEVEL: "16" script: - sycl-ls - export ONEAPI_DEVICE_SELECTOR=level_zero:gpu + # configure and build through CTest, so that CDash records their timings + # as well as compiler warnings and errors - | - cmake -S . -B build \ + Tools/CI/ctest_dashboard.sh build \ -DBUILD_SHARED_LIBS=ON \ - -DBUILDNAME=GPU-PVC-SYCL-MP \ - -DSITE=Frank_UO \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_C_COMPILER=$(which icx) \ -DCMAKE_CXX_COMPILER=$(which icpx) \ @@ -129,14 +140,8 @@ Intel-PVC: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ -DWarpX_PARTICLE_PRECISION=SINGLE - - cmake --build build -j 16 - export AMREX_DEFAULT_INIT="amrex.the_arena_init_size=0" - | - if [[ "${CI_COMMIT_BRANCH}" == "development" ]]; then - # run tests and submit results to CDash as Experimental - ctest --test-dir build -R test_3d_laser_acceleration_btd \ - --output-on-failure --no-tests=error -D ExperimentalTest -D ExperimentalSubmit - else - ctest --test-dir build -R test_3d_laser_acceleration_btd \ - --output-on-failure --no-tests=error - fi + ctest --test-dir build -R test_3d_laser_acceleration_btd \ + --output-on-failure --no-tests=error -D ExperimentalTest + after_script: *cdash-submit diff --git a/AGENTS.md b/AGENTS.md index ccf44bd35bc..2a0f15ec46e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,17 @@ Test output goes to `build/bin//`. runners, through a GitHub-to-GitLab mirror. Only triggered on merges to `development` and on PRs labeled `bot: run GPU`, so it is absent from a default PR's checks. +### CDash Dashboard + +CI publishes to [CDash](https://my.cdash.org/index.php?project=WarpX). Configure and build results +only reach it when those steps run *through* CTest, so CI configures and builds via +`Tools/CI/ctest_dashboard.sh` (a wrapper around `Tools/CI/ctest_dashboard.cmake`) rather than +calling `cmake` and `cmake --build` directly. The test step stays a plain `ctest` call with +`-D ExperimentalTest` appended, and one `-D ExperimentalSubmit` uploads all parts as one build. +Set `CDASH_BUILD_NAME` and `CDASH_SITE` per job; the build parallelism comes from +`CMAKE_BUILD_PARALLEL_LEVEL`, since the CTest build step has no `-j`. See +`Docs/source/developers/how_to_test.rst` for details. + ### Adding a Test Use `add_warpx_test()` in the test directory's `CMakeLists.txt`. Generate checksums with `CHECKSUM_RESET=ON ctest --test-dir build -R your_test_name`. diff --git a/Docs/source/developers/how_to_test.rst b/Docs/source/developers/how_to_test.rst index e1995db3658..b4412434509 100644 --- a/Docs/source/developers/how_to_test.rst +++ b/Docs/source/developers/how_to_test.rst @@ -221,6 +221,56 @@ Alternatively, it is also possible to reset a checksum file locally by running t Note that it is possible that the checksum values generated locally on your computer architecture may differ from the ones generated remotely by the autometed tests on the architecture provided by the CI runners. +.. _developers-testing-cdash: + +How CI results reach the CDash dashboard +---------------------------------------- + +Our CI runners publish their results to the `WarpX dashboard on CDash `__. +A dashboard build collects three kinds of results, each produced by a separate CTest step: + +* **Configure** (``Configure.xml``): how long ``cmake`` took, plus any CMake warnings and errors. +* **Build** (``Build.xml``): how long the compilation took, plus every compiler warning and error with its source file and line. +* **Test** (``Test.xml``): the pass/fail status and runtime of each test. + +CDash only receives the configure and build results if those steps run *through* CTest. +Calling ``cmake`` and ``cmake --build`` directly and then submitting only produces a ``Test.xml``. +Our CI therefore configures and builds via the helper script +`Tools/CI/ctest_dashboard.sh `__, +which drives ``ctest_configure()`` and ``ctest_build()`` from a CTest script: + +.. code-block:: bash + + CDASH_BUILD_NAME=CPU-3D CDASH_SITE=Azure CMAKE_BUILD_PARALLEL_LEVEL=2 \ + Tools/CI/ctest_dashboard.sh build -DWarpX_DIMS=3 -DWarpX_FFT=ON + +The CMake options are passed through unchanged. +Note that the build parallelism comes from ``CMAKE_BUILD_PARALLEL_LEVEL``: the CTest build step has no ``-j`` option of its own. + +The script leaves the test step to the caller, so that CI keeps using the plain ``ctest`` command line it already has, adding only ``-D ExperimentalTest``: + +.. code-block:: bash + + ctest --test-dir build --output-on-failure -E AMReX -D ExperimentalTest + +All steps write into the same dashboard tag, so one final call uploads them as a single CDash build: + +.. code-block:: bash + + ctest --test-dir build -D ExperimentalSubmit + +Two conventions keep the dashboard readable: + +* Every build gets a unique ``CDASH_BUILD_NAME``, and ``CDASH_SITE`` names the CI provider + (``Azure``, ``GitHub-Actions`` or ``Frank_UO`` for the GPU runners). + A job that configures more than one build directory sets a distinct name per directory. +* Only merges to ``development`` are submitted. + Pull request runs perform the same steps and write the XML files, but drop them. + +Build-only jobs, such as the compile matrix in ``.github/workflows/``, submit from the script itself by setting ``CDASH_SUBMIT=ON``. +Jobs that also run tests leave it ``OFF`` and submit once at the end, from a step that runs even when the build or the tests failed -- a broken build is exactly what the dashboard should show. + + .. _developers-testing-naming: Naming conventions for automated tests diff --git a/Tools/CI/ctest_dashboard.cmake b/Tools/CI/ctest_dashboard.cmake new file mode 100644 index 00000000000..7b88bd97a2d --- /dev/null +++ b/Tools/CI/ctest_dashboard.cmake @@ -0,0 +1,96 @@ +# CTest driver script for CDash submissions from CI. +# +# CDash only reports configure and build results -- timings, compiler warnings +# and errors -- if those steps run *through* CTest, which records them in +# Configure.xml and Build.xml. Calling `cmake` and `cmake --build` directly and +# then running `ctest -D ExperimentalTest` produces a Test.xml only, which is +# why our dashboard used to show test results and nothing else. +# +# This script runs the configure and build steps via ctest_configure() and +# ctest_build(). The test step is deliberately left to the caller, so that CI +# keeps using the plain `ctest` command line it already has (including options +# such as `--no-tests=error`, which ctest_test() does not offer). Every step +# writes into the same dashboard tag created by ctest_start() here, so a single +# `ctest -D ExperimentalSubmit` at the end uploads all parts as one build. +# +# Do not invoke this file directly, use Tools/CI/ctest_dashboard.sh. +# +# Environment: +# CDASH_SOURCE_DIR top-level WarpX source directory (required) +# CDASH_BINARY_DIR build directory (required) +# CDASH_BUILD_NAME dashboard build name, e.g. "CPU-3D" (required) +# CDASH_SITE dashboard site name, e.g. "Azure" (required) +# CDASH_OPTIONS_FILE file with one CMake configure option per line +# CDASH_SUBMIT "ON" to submit right after the build (build-only jobs) +# CMAKE_GENERATOR CMake generator, unless given as -G in the options +# +# `cmake --build` honors CMAKE_BUILD_PARALLEL_LEVEL, and CTest builds via +# `cmake --build`, so the build parallelism is inherited from the environment. + +cmake_minimum_required(VERSION 3.24) + +foreach(_required IN ITEMS CDASH_SOURCE_DIR CDASH_BINARY_DIR CDASH_BUILD_NAME CDASH_SITE) + if("$ENV{${_required}}" STREQUAL "") + message(FATAL_ERROR "ctest_dashboard: environment variable ${_required} is not set") + endif() +endforeach() + +set(CTEST_SOURCE_DIRECTORY "$ENV{CDASH_SOURCE_DIR}") +set(CTEST_BINARY_DIRECTORY "$ENV{CDASH_BINARY_DIR}") +set(CTEST_BUILD_NAME "$ENV{CDASH_BUILD_NAME}") +set(CTEST_SITE "$ENV{CDASH_SITE}") + +# Configure options are handed over one per line in a file, so that values +# containing spaces (-DCMAKE_CXX_FLAGS="-Werror -Wall") or semicolons +# (-DWarpX_DIMS="1;2") survive without any shell or CMake list quoting games. +set(configure_options "") +if(NOT "$ENV{CDASH_OPTIONS_FILE}" STREQUAL "") + file(STRINGS "$ENV{CDASH_OPTIONS_FILE}" raw_options) + foreach(option IN LISTS raw_options) + # escape embedded ";" so that each line stays a single list element + string(REPLACE ";" "\;" option "${option}") + list(APPEND configure_options "${option}") + endforeach() +endif() + +# ctest_configure() refuses to run without a generator, and appends -G after the +# options we pass. An explicit -G among them therefore has to be picked up here, +# or our own -G would silently override it. Both "-GNinja" and "-G;Ninja" occur. +set(CTEST_CMAKE_GENERATOR "$ENV{CMAKE_GENERATOR}") +set(next_is_generator FALSE) +foreach(option IN LISTS configure_options) + if(next_is_generator) + set(CTEST_CMAKE_GENERATOR "${option}") + set(next_is_generator FALSE) + elseif(option STREQUAL "-G") + set(next_is_generator TRUE) + elseif(option MATCHES "^-G(.+)$") + set(CTEST_CMAKE_GENERATOR "${CMAKE_MATCH_1}") + endif() +endforeach() +if(CTEST_CMAKE_GENERATOR STREQUAL "") + # same default as a plain `cmake` call on the platforms we submit from + set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +endif() + +ctest_start(Experimental) + +ctest_configure(OPTIONS "${configure_options}" RETURN_VALUE configure_rv) + +# A failed configure leaves nothing to build, but Configure.xml is still worth +# submitting: it carries the CMake error that broke the job. +set(build_rv 0) +if(configure_rv EQUAL 0) + ctest_build(RETURN_VALUE build_rv) +endif() + +if("$ENV{CDASH_SUBMIT}" STREQUAL "ON") + ctest_submit(RETRY_COUNT 3 RETRY_DELAY 15) +endif() + +if(NOT configure_rv EQUAL 0) + message(FATAL_ERROR "ctest_dashboard: configure failed") +endif() +if(NOT build_rv EQUAL 0) + message(FATAL_ERROR "ctest_dashboard: build failed") +endif() diff --git a/Tools/CI/ctest_dashboard.sh b/Tools/CI/ctest_dashboard.sh new file mode 100755 index 00000000000..c6b4e437f65 --- /dev/null +++ b/Tools/CI/ctest_dashboard.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Configure and build WarpX through CTest, so that CDash records configure and +# build timings as well as compiler warnings and errors. +# +# Usage: +# Tools/CI/ctest_dashboard.sh [cmake option ...] +# +# The CMake options are passed to the configure step unchanged; quote them as +# usual, values with spaces or semicolons are preserved. +# +# Environment: +# CDASH_BUILD_NAME dashboard build name, e.g. "CPU-3D" (required) +# CDASH_SITE dashboard site name, e.g. "Azure" (required) +# CDASH_SUBMIT "ON" submits to CDash right away; use this for build-only +# jobs. Jobs that also run tests should leave it "OFF" and +# submit once at the end, after their +# `ctest ... -D ExperimentalTest` step, with +# `ctest --test-dir -D ExperimentalSubmit`. +# +# The build parallelism comes from CMAKE_BUILD_PARALLEL_LEVEL: CTest builds via +# `cmake --build`, which honors it. There is no `-j` for the CTest build step. + +set -o nounset -o errexit -o pipefail + +if [ $# -lt 1 ]; then + echo "Usage: $(basename "$0") [cmake option ...]" >&2 + exit 2 +fi + +build_dir="$1" +shift + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source_dir="$(cd -- "${script_dir}/../.." && pwd)" + +mkdir -p "${build_dir}" +CDASH_BINARY_DIR="$(cd -- "${build_dir}" && pwd)" +export CDASH_BINARY_DIR + +export CDASH_SOURCE_DIR="${source_dir}" +export CDASH_BUILD_NAME="${CDASH_BUILD_NAME:?CDASH_BUILD_NAME must be set}" +export CDASH_SITE="${CDASH_SITE:?CDASH_SITE must be set}" +export CDASH_SUBMIT="${CDASH_SUBMIT:-OFF}" + +# hand the options over in a file: one per line, no quoting games needed +CDASH_OPTIONS_FILE="${CDASH_BINARY_DIR}/.cdash_configure_options" +export CDASH_OPTIONS_FILE +: > "${CDASH_OPTIONS_FILE}" +if [ $# -gt 0 ]; then + printf '%s\n' "$@" > "${CDASH_OPTIONS_FILE}" +fi +# BUILDNAME and SITE are what include(CTest) writes into DartConfiguration.tcl, +# from where a later `ctest -D ExperimentalTest` picks them up for Test.xml. +# Without them that step would label Test.xml with the host name and a generic +# build name, and CDash would file it as a build separate from Configure.xml and +# Build.xml. Appended last so they always match CTEST_BUILD_NAME/CTEST_SITE. +printf '%s\n' "-DBUILDNAME=${CDASH_BUILD_NAME}" "-DSITE=${CDASH_SITE}" \ + >> "${CDASH_OPTIONS_FILE}" + +exec ctest -VV -S "${script_dir}/ctest_dashboard.cmake" From e2a0517b8e33c9c1f0925e440b3c5ad27c396b40 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 4 Sep 2026 11:01:01 -0700 Subject: [PATCH 2/4] CI: harden the CDash dashboard driver Follow-up to the previous commit, from review of the CDash wiring. Each item was reproduced against CTest 3.31 before and after the fix. - Submit a configure that fails before `include(CTest)`. Such a failure leaves no `DartConfiguration.tcl`, so the CI's own `ctest -D ExperimentalSubmit` aborts with exit 64 and discards the `Configure.xml` holding the CMake error. Everything above `CMakeLists.txt:199` is in that window: compiler detection, the `WarpX_DIMS` validation, all `option()` handling -- exactly the failures a dashboard is most useful for. The driver now submits these itself, driven by `CTestConfig.cmake` rather than the build directory, and the CI submit steps skip when `DartConfiguration.tcl` is absent. - Stop a CDash outage from failing a green build. `ctest_submit()` made `ctest -S` exit 255 on an upload error, which on pushes to `development` would fail all 14 GitHub build jobs whose builds had already succeeded. Note that `RETURN_VALUE` alone does not prevent this; `CAPTURE_CMAKE_ERROR` is what does. - Pass the build parallelism to `ctest_build(PARALLEL_LEVEL ...)` explicitly. It was reaching the compiler only by leaking through the environment, which also silently parallelized the later `--target pip_install` calls in the macOS and Intel jobs. An unset `CMAKE_BUILD_PARALLEL_LEVEL` now passes nothing rather than forcing a level, so Ninja keeps its own one-job-per-core default. - Restore `build_pyfull`'s build scope via the new `CDASH_BUILD_TARGET`. That job only ever built the `pip_install` target, which does not depend on the `warpx` app, so building the default target added a compile and link the job never did. - Correct the claim that the CTest build step has no `-j`, repeated in four places. `ctest_build()` does take `PARALLEL_LEVEL`. - Note that `ctest_start()` reuses a tag only within the same UTC day, so an Azure job straddling midnight splits into two dashboard builds. - `file(STRINGS ENCODING UTF-8)` so a non-ASCII option cannot be silently truncated, and require CMake 3.25 to match the project. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CxAc564RY4pPkGbhio7fBm --- .azure-pipelines.yml | 4 +- .github/workflows/ubuntu.yml | 4 +- .gitlab/ci.yaml | 4 +- AGENTS.md | 4 +- Docs/source/developers/how_to_test.rst | 4 +- Tools/CI/ctest_dashboard.cmake | 51 +++++++++++++++++++++----- Tools/CI/ctest_dashboard.sh | 9 ++++- 7 files changed, 62 insertions(+), 18 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index cbc55e26590..a4ebb3121c6 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -220,7 +220,9 @@ jobs: # the XML but keep it. - bash: | set -o nounset errexit pipefail - if [[ "$(Build.SourceBranch)" == "refs/heads/development" ]]; then + # DartConfiguration.tcl is missing when the configure died before + # include(CTest); the driver script has already submitted in that case. + if [[ "$(Build.SourceBranch)" == "refs/heads/development" && -f build/DartConfiguration.tcl ]]; then ctest --test-dir build -D ExperimentalSubmit fi displayName: 'CDash Submit' diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9d1926735d4..b58b127ab53 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -234,12 +234,14 @@ jobs: export CXXFLAGS="-Werror -Wno-error=pass-failed" + # pip_install does not depend on the warpx app, so building the + # default target here would add a compile+link this job never did + CDASH_BUILD_TARGET=pip_install \ Tools/CI/ctest_dashboard.sh build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 4 --target pip_install ccache -s du -hs ~/.cache/ccache diff --git a/.gitlab/ci.yaml b/.gitlab/ci.yaml index 87d740ff713..acb83145418 100644 --- a/.gitlab/ci.yaml +++ b/.gitlab/ci.yaml @@ -34,7 +34,9 @@ # merges to `development` are submitted; PR pipelines write the XML but keep it. .cdash-submit: &cdash-submit - | - if [[ "${CI_COMMIT_BRANCH}" == "development" ]]; then + # DartConfiguration.tcl is missing when the configure died before + # include(CTest); the driver script has already submitted in that case. + if [[ "${CI_COMMIT_BRANCH}" == "development" && -f build/DartConfiguration.tcl ]]; then ctest --test-dir build -D ExperimentalSubmit fi diff --git a/AGENTS.md b/AGENTS.md index 2a0f15ec46e..ffcd34894d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,8 +90,8 @@ only reach it when those steps run *through* CTest, so CI configures and builds `Tools/CI/ctest_dashboard.sh` (a wrapper around `Tools/CI/ctest_dashboard.cmake`) rather than calling `cmake` and `cmake --build` directly. The test step stays a plain `ctest` call with `-D ExperimentalTest` appended, and one `-D ExperimentalSubmit` uploads all parts as one build. -Set `CDASH_BUILD_NAME` and `CDASH_SITE` per job; the build parallelism comes from -`CMAKE_BUILD_PARALLEL_LEVEL`, since the CTest build step has no `-j`. See +Set `CDASH_BUILD_NAME` and `CDASH_SITE` per job; build parallelism comes from +`CMAKE_BUILD_PARALLEL_LEVEL` and the target from the optional `CDASH_BUILD_TARGET`. See `Docs/source/developers/how_to_test.rst` for details. ### Adding a Test diff --git a/Docs/source/developers/how_to_test.rst b/Docs/source/developers/how_to_test.rst index b4412434509..c2d441a844e 100644 --- a/Docs/source/developers/how_to_test.rst +++ b/Docs/source/developers/how_to_test.rst @@ -245,7 +245,8 @@ which drives ``ctest_configure()`` and ``ctest_build()`` from a CTest script: Tools/CI/ctest_dashboard.sh build -DWarpX_DIMS=3 -DWarpX_FFT=ON The CMake options are passed through unchanged. -Note that the build parallelism comes from ``CMAKE_BUILD_PARALLEL_LEVEL``: the CTest build step has no ``-j`` option of its own. +The build parallelism comes from ``CMAKE_BUILD_PARALLEL_LEVEL``, which the script passes on to ``ctest_build(PARALLEL_LEVEL ...)``. +Set ``CDASH_BUILD_TARGET`` to build a specific target instead of the default one. The script leaves the test step to the caller, so that CI keeps using the plain ``ctest`` command line it already has, adding only ``-D ExperimentalTest``: @@ -269,6 +270,7 @@ Two conventions keep the dashboard readable: Build-only jobs, such as the compile matrix in ``.github/workflows/``, submit from the script itself by setting ``CDASH_SUBMIT=ON``. Jobs that also run tests leave it ``OFF`` and submit once at the end, from a step that runs even when the build or the tests failed -- a broken build is exactly what the dashboard should show. +A configure that fails before ``include(CTest)`` is an exception: ``ctest -D ExperimentalSubmit`` needs a ``DartConfiguration.tcl`` that does not exist yet at that point, so the script submits such a failure itself. .. _developers-testing-naming: diff --git a/Tools/CI/ctest_dashboard.cmake b/Tools/CI/ctest_dashboard.cmake index 7b88bd97a2d..f2b29ceb4fb 100644 --- a/Tools/CI/ctest_dashboard.cmake +++ b/Tools/CI/ctest_dashboard.cmake @@ -21,13 +21,12 @@ # CDASH_BUILD_NAME dashboard build name, e.g. "CPU-3D" (required) # CDASH_SITE dashboard site name, e.g. "Azure" (required) # CDASH_OPTIONS_FILE file with one CMake configure option per line +# CDASH_BUILD_TARGET build this target instead of the default one # CDASH_SUBMIT "ON" to submit right after the build (build-only jobs) # CMAKE_GENERATOR CMake generator, unless given as -G in the options -# -# `cmake --build` honors CMAKE_BUILD_PARALLEL_LEVEL, and CTest builds via -# `cmake --build`, so the build parallelism is inherited from the environment. +# CMAKE_BUILD_PARALLEL_LEVEL build parallelism, passed on to ctest_build() -cmake_minimum_required(VERSION 3.24) +cmake_minimum_required(VERSION 3.25) foreach(_required IN ITEMS CDASH_SOURCE_DIR CDASH_BINARY_DIR CDASH_BUILD_NAME CDASH_SITE) if("$ENV{${_required}}" STREQUAL "") @@ -45,7 +44,8 @@ set(CTEST_SITE "$ENV{CDASH_SITE}") # (-DWarpX_DIMS="1;2") survive without any shell or CMake list quoting games. set(configure_options "") if(NOT "$ENV{CDASH_OPTIONS_FILE}" STREQUAL "") - file(STRINGS "$ENV{CDASH_OPTIONS_FILE}" raw_options) + # ENCODING: file(STRINGS) otherwise drops non-ASCII bytes silently + file(STRINGS "$ENV{CDASH_OPTIONS_FILE}" raw_options ENCODING UTF-8) foreach(option IN LISTS raw_options) # escape embedded ";" so that each line stays a single list element string(REPLACE ";" "\;" option "${option}") @@ -73,19 +73,50 @@ if(CTEST_CMAKE_GENERATOR STREQUAL "") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") endif() +# Pass the parallelism to ctest_build() explicitly rather than relying on it +# leaking through the environment into `cmake --build`. When it is unset, pass +# nothing at all: forcing a level here would drop Ninja from its own default of +# one job per core down to whatever we picked. +set(build_parallel_arg "") +if("$ENV{CMAKE_BUILD_PARALLEL_LEVEL}" MATCHES "^[1-9][0-9]*$") + set(build_parallel_arg PARALLEL_LEVEL "$ENV{CMAKE_BUILD_PARALLEL_LEVEL}") +endif() + +# ctest_start() reuses an existing Testing/TAG only while its date still matches +# today's in UTC. A job that configures before and tests after midnight UTC +# therefore splits into two dashboard builds; rare, and it heals on the next run. + ctest_start(Experimental) ctest_configure(OPTIONS "${configure_options}" RETURN_VALUE configure_rv) -# A failed configure leaves nothing to build, but Configure.xml is still worth -# submitting: it carries the CMake error that broke the job. +# A failed configure leaves nothing to build. set(build_rv 0) if(configure_rv EQUAL 0) - ctest_build(RETURN_VALUE build_rv) + if(NOT "$ENV{CDASH_BUILD_TARGET}" STREQUAL "") + ctest_build(TARGET "$ENV{CDASH_BUILD_TARGET}" + ${build_parallel_arg} RETURN_VALUE build_rv) + else() + ctest_build(${build_parallel_arg} RETURN_VALUE build_rv) + endif() endif() -if("$ENV{CDASH_SUBMIT}" STREQUAL "ON") - ctest_submit(RETRY_COUNT 3 RETRY_DELAY 15) +# Submit when the caller asked for it, and *always* when the configure failed. +# A caller that submits from its own `ctest -D ExperimentalSubmit` step cannot +# do it in that case: that command reads the build directory through +# DartConfiguration.tcl, which include(CTest) only writes once the configure has +# got that far, so it would abort and discard the Configure.xml holding the +# CMake error. Here the submit is driven by CTestConfig.cmake in the source +# directory instead, and works no matter how early the configure died. +if("$ENV{CDASH_SUBMIT}" OR NOT configure_rv EQUAL 0) + # Never let the dashboard decide whether the build passed: a CDash outage + # must not turn a green build red. RETURN_VALUE alone does not do that -- + # CAPTURE_CMAKE_ERROR is what keeps ctest from exiting non-zero. + ctest_submit(RETRY_COUNT 3 RETRY_DELAY 15 + RETURN_VALUE submit_rv CAPTURE_CMAKE_ERROR submit_err) + if(NOT submit_rv EQUAL 0 OR NOT submit_err EQUAL 0) + message("ctest_dashboard: submission to CDash failed, continuing anyway") + endif() endif() if(NOT configure_rv EQUAL 0) diff --git a/Tools/CI/ctest_dashboard.sh b/Tools/CI/ctest_dashboard.sh index c6b4e437f65..93e014a9269 100755 --- a/Tools/CI/ctest_dashboard.sh +++ b/Tools/CI/ctest_dashboard.sh @@ -17,9 +17,13 @@ # submit once at the end, after their # `ctest ... -D ExperimentalTest` step, with # `ctest --test-dir -D ExperimentalSubmit`. +# A failed configure is always submitted from here, because +# that later command needs a DartConfiguration.tcl which +# does not exist yet when the configure dies early. +# CDASH_BUILD_TARGET build this target instead of the default one # -# The build parallelism comes from CMAKE_BUILD_PARALLEL_LEVEL: CTest builds via -# `cmake --build`, which honors it. There is no `-j` for the CTest build step. +# The build parallelism comes from CMAKE_BUILD_PARALLEL_LEVEL, which is passed +# on to ctest_build(PARALLEL_LEVEL). set -o nounset -o errexit -o pipefail @@ -42,6 +46,7 @@ export CDASH_SOURCE_DIR="${source_dir}" export CDASH_BUILD_NAME="${CDASH_BUILD_NAME:?CDASH_BUILD_NAME must be set}" export CDASH_SITE="${CDASH_SITE:?CDASH_SITE must be set}" export CDASH_SUBMIT="${CDASH_SUBMIT:-OFF}" +export CDASH_BUILD_TARGET="${CDASH_BUILD_TARGET:-}" # hand the options over in a file: one per line, no quoting games needed CDASH_OPTIONS_FILE="${CDASH_BINARY_DIR}/.cdash_configure_options" From 7283b8f8537c2ef29850f52743a3f5383f965367 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 4 Sep 2026 16:01:19 -0700 Subject: [PATCH 3/4] CI: keep pip out of the CTest build step The `Clang pywarpx` job failed on this branch. The build itself was fine -- `pip` reported "Successfully installed pywarpx-26.9" and the build command exited 0 -- but CTest reported "15 Compiler errors" and the job exited 255. CTest finds build errors by matching regexes against the build log, and one of its default patterns is `([^ :]+):([0-9]+): ([^ \t])`. Every one of the 15 "errors" was a Python warning from `pip`: setuptools/_distutils/dist.py:318: UserWarning: Unknown distribution option: 'tests_require' setuptools/dist.py:332: InformationOnly: Normalizing '26.09' to '26.9' CTest's exception list covers `: warning` but not setuptools' warning class names, so these are counted as errors. Applying the default matchers to the job log by hand reproduces the count exactly, 15 of 15. The cause was `CDASH_BUILD_TARGET=pip_install`, added in the previous commit to stop the dashboard building targets this job never built. That pulled pip's output inside the build step CTest scrapes. `build_pyfull` now builds the default target and runs `--target pip_install` afterwards, outside the dashboard step -- the same shape as the macOS and Intel jobs, which pass. The cost is the extra app compile and link that the previous commit set out to avoid; roughly a minute, and worth it here. `CDASH_BUILD_TARGET` is removed rather than left unused: its only purpose was this job, and a knob whose obvious use is a packaging target is a trap. The restriction is now documented where the script builds. Note that the concurrently failing `clang-tidy-1D` job is unrelated to this branch: `apt.llvm.org` was unreachable while installing LLVM ("connect (101: Network is unreachable)"). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CxAc564RY4pPkGbhio7fBm --- .github/workflows/ubuntu.yml | 7 ++++--- AGENTS.md | 5 +++-- Docs/source/developers/how_to_test.rst | 4 +++- Tools/CI/ctest_dashboard.cmake | 11 ++++------- Tools/CI/ctest_dashboard.sh | 2 -- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index b58b127ab53..7aef8d8d209 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -234,14 +234,15 @@ jobs: export CXXFLAGS="-Werror -Wno-error=pass-failed" - # pip_install does not depend on the warpx app, so building the - # default target here would add a compile+link this job never did - CDASH_BUILD_TARGET=pip_install \ Tools/CI/ctest_dashboard.sh build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON + # deliberately outside the dashboard build: CTest scrapes the build log + # for compiler errors, and pip's setuptools warnings + # ("dist.py:318: UserWarning: ...") match its error pattern + cmake --build build -j 4 --target pip_install ccache -s du -hs ~/.cache/ccache diff --git a/AGENTS.md b/AGENTS.md index ffcd34894d6..738b012d08d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,8 +91,9 @@ only reach it when those steps run *through* CTest, so CI configures and builds calling `cmake` and `cmake --build` directly. The test step stays a plain `ctest` call with `-D ExperimentalTest` appended, and one `-D ExperimentalSubmit` uploads all parts as one build. Set `CDASH_BUILD_NAME` and `CDASH_SITE` per job; build parallelism comes from -`CMAKE_BUILD_PARALLEL_LEVEL` and the target from the optional `CDASH_BUILD_TARGET`. See -`Docs/source/developers/how_to_test.rst` for details. +`CMAKE_BUILD_PARALLEL_LEVEL`. The script builds the default target only — keep +`--target pip_install` outside it, since CTest reads pip's setuptools warnings as +build errors. See `Docs/source/developers/how_to_test.rst` for details. ### Adding a Test diff --git a/Docs/source/developers/how_to_test.rst b/Docs/source/developers/how_to_test.rst index c2d441a844e..ed83a0439b2 100644 --- a/Docs/source/developers/how_to_test.rst +++ b/Docs/source/developers/how_to_test.rst @@ -246,7 +246,9 @@ which drives ``ctest_configure()`` and ``ctest_build()`` from a CTest script: The CMake options are passed through unchanged. The build parallelism comes from ``CMAKE_BUILD_PARALLEL_LEVEL``, which the script passes on to ``ctest_build(PARALLEL_LEVEL ...)``. -Set ``CDASH_BUILD_TARGET`` to build a specific target instead of the default one. + +The script always builds the default target. +Keep packaging steps such as ``--target pip_install`` outside of it: CTest scrapes the build log for compiler errors, and ``pip``'s setuptools warnings (``dist.py:318: UserWarning: ...``) match its error pattern, which would fail the job on a build that actually succeeded. The script leaves the test step to the caller, so that CI keeps using the plain ``ctest`` command line it already has, adding only ``-D ExperimentalTest``: diff --git a/Tools/CI/ctest_dashboard.cmake b/Tools/CI/ctest_dashboard.cmake index f2b29ceb4fb..283b3d3daf5 100644 --- a/Tools/CI/ctest_dashboard.cmake +++ b/Tools/CI/ctest_dashboard.cmake @@ -21,7 +21,6 @@ # CDASH_BUILD_NAME dashboard build name, e.g. "CPU-3D" (required) # CDASH_SITE dashboard site name, e.g. "Azure" (required) # CDASH_OPTIONS_FILE file with one CMake configure option per line -# CDASH_BUILD_TARGET build this target instead of the default one # CDASH_SUBMIT "ON" to submit right after the build (build-only jobs) # CMAKE_GENERATOR CMake generator, unless given as -G in the options # CMAKE_BUILD_PARALLEL_LEVEL build parallelism, passed on to ctest_build() @@ -93,12 +92,10 @@ ctest_configure(OPTIONS "${configure_options}" RETURN_VALUE configure_rv) # A failed configure leaves nothing to build. set(build_rv 0) if(configure_rv EQUAL 0) - if(NOT "$ENV{CDASH_BUILD_TARGET}" STREQUAL "") - ctest_build(TARGET "$ENV{CDASH_BUILD_TARGET}" - ${build_parallel_arg} RETURN_VALUE build_rv) - else() - ctest_build(${build_parallel_arg} RETURN_VALUE build_rv) - endif() + # Only ever the default target: CTest scrapes the build log for compiler + # errors, so a packaging target that shells out to pip would report its + # setuptools warnings as build errors. + ctest_build(${build_parallel_arg} RETURN_VALUE build_rv) endif() # Submit when the caller asked for it, and *always* when the configure failed. diff --git a/Tools/CI/ctest_dashboard.sh b/Tools/CI/ctest_dashboard.sh index 93e014a9269..f5ed88973b9 100755 --- a/Tools/CI/ctest_dashboard.sh +++ b/Tools/CI/ctest_dashboard.sh @@ -20,7 +20,6 @@ # A failed configure is always submitted from here, because # that later command needs a DartConfiguration.tcl which # does not exist yet when the configure dies early. -# CDASH_BUILD_TARGET build this target instead of the default one # # The build parallelism comes from CMAKE_BUILD_PARALLEL_LEVEL, which is passed # on to ctest_build(PARALLEL_LEVEL). @@ -46,7 +45,6 @@ export CDASH_SOURCE_DIR="${source_dir}" export CDASH_BUILD_NAME="${CDASH_BUILD_NAME:?CDASH_BUILD_NAME must be set}" export CDASH_SITE="${CDASH_SITE:?CDASH_SITE must be set}" export CDASH_SUBMIT="${CDASH_SUBMIT:-OFF}" -export CDASH_BUILD_TARGET="${CDASH_BUILD_TARGET:-}" # hand the options over in a file: one per line, no quoting games needed CDASH_OPTIONS_FILE="${CDASH_BINARY_DIR}/.cdash_configure_options" From 5a019e0f25c3f561ed9b8c07f97e7a4ee3aa794e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 4 Sep 2026 16:39:03 -0700 Subject: [PATCH 4/4] CI: simplify the CDash dashboard driver Review of the plumbing, not the design. `ctest -S` stays: the CTest command line drives an already-generated build tree, so it can only time a warm re-configure, and a dashboard client script is the documented way to drive the configure itself. It is also what VTK, ParaView, CMake, ITK, Trilinos and deal.II do. The wrapper around it was doing work CMake already does: - `separate_arguments(UNIX_COMMAND)` parses shell-quoted arguments back into one list element each and escapes embedded `;` on the way. That replaces the whole options-file mechanism: the temporary file, the `file(STRINGS ENCODING UTF-8)` read and the `string(REPLACE ";" "\;")` loop. The wrapper now hands the options over with `printf '%q '`. - `CTEST_SCRIPT_DIRECTORY` gives the source directory, so the shell no longer resolves it. - `list(APPEND)` adds `-DBUILDNAME`/`-DSITE` in one line. - The 15-line `-G` parser, which existed because `ctest_configure()` appends its own generator after our options and would override one passed there, becomes a guard that says so. The two jobs that passed `-GNinja` inline now set `CMAKE_GENERATOR`, like every other job. 81 -> 62 lines of actual code, and three mechanisms fewer to understand. No behaviour change and no CI YAML churn beyond those two `-GNinja`. Re-verified after the rewrite: option fidelity for values with spaces and semicolons, all four generator paths, parallelism set/unset/malformed, identical BuildName and Site across Configure.xml, Build.xml and Test.xml, and the exit codes -- success 0, build failure 255, early configure failure 255 with Configure.xml still uploaded, CDash unreachable 0. Considered and rejected: `CMakePresets.json`. Workflow presets produce no dashboard XML at all, test presets have no dashboard schema, and `ctest --preset X -D ExperimentalTest` ignores the preset's `binaryDir`, finds no tests and exits 0 -- green CI that tested nothing. Presets are worth having for local developer ergonomics, separately. `CTEST_USE_LAUNCHERS` would give exact per-file diagnostics instead of regex log scraping, but a signal-killed custom command passes silently under launchers until CMake 4.5 (cmake#27921), so it wants its own PR. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CxAc564RY4pPkGbhio7fBm --- .github/workflows/ubuntu.yml | 4 +- AGENTS.md | 2 +- Docs/source/developers/how_to_test.rst | 4 +- Tools/CI/ctest_dashboard.cmake | 95 +++++++++++++------------- Tools/CI/ctest_dashboard.sh | 53 ++++---------- 5 files changed, 66 insertions(+), 92 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 7aef8d8d209..af8fa38bb4c 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -75,6 +75,7 @@ jobs: CC: "gcc-13" CDASH_BUILD_NAME: Linux-GCC13-1D2D CMAKE_BUILD_PARALLEL_LEVEL: 4 + CMAKE_GENERATOR: Ninja steps: - uses: actions/checkout@v7 - name: install dependencies @@ -95,7 +96,6 @@ jobs: ccache -z Tools/CI/ctest_dashboard.sh build \ - -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="1;2" \ -DWarpX_EB=OFF \ @@ -122,6 +122,7 @@ jobs: CC: "gcc-13" CDASH_BUILD_NAME: Linux-GCC13-RZ3D-SP CMAKE_BUILD_PARALLEL_LEVEL: 4 + CMAKE_GENERATOR: Ninja steps: - uses: actions/checkout@v7 - name: install dependencies @@ -146,7 +147,6 @@ jobs: export CXXFLAGS="-Werror" Tools/CI/ctest_dashboard.sh build \ - -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ -DWarpX_EB=OFF \ diff --git a/AGENTS.md b/AGENTS.md index 738b012d08d..f0d1ec914fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,7 +91,7 @@ only reach it when those steps run *through* CTest, so CI configures and builds calling `cmake` and `cmake --build` directly. The test step stays a plain `ctest` call with `-D ExperimentalTest` appended, and one `-D ExperimentalSubmit` uploads all parts as one build. Set `CDASH_BUILD_NAME` and `CDASH_SITE` per job; build parallelism comes from -`CMAKE_BUILD_PARALLEL_LEVEL`. The script builds the default target only — keep +`CMAKE_BUILD_PARALLEL_LEVEL` and the generator from `CMAKE_GENERATOR` (not `-G`). The script builds the default target only — keep `--target pip_install` outside it, since CTest reads pip's setuptools warnings as build errors. See `Docs/source/developers/how_to_test.rst` for details. diff --git a/Docs/source/developers/how_to_test.rst b/Docs/source/developers/how_to_test.rst index ed83a0439b2..694c9e87578 100644 --- a/Docs/source/developers/how_to_test.rst +++ b/Docs/source/developers/how_to_test.rst @@ -235,6 +235,7 @@ A dashboard build collects three kinds of results, each produced by a separate C CDash only receives the configure and build results if those steps run *through* CTest. Calling ``cmake`` and ``cmake --build`` directly and then submitting only produces a ``Test.xml``. +The command line route (``ctest -D ExperimentalConfigure``) is not an option here: it drives an already-generated build tree and needs a ``DartConfiguration.tcl``, which ``include(CTest)`` only writes during a configure, so it could only ever time a *re*-configure. Our CI therefore configures and builds via the helper script `Tools/CI/ctest_dashboard.sh `__, which drives ``ctest_configure()`` and ``ctest_build()`` from a CTest script: @@ -245,7 +246,8 @@ which drives ``ctest_configure()`` and ``ctest_build()`` from a CTest script: Tools/CI/ctest_dashboard.sh build -DWarpX_DIMS=3 -DWarpX_FFT=ON The CMake options are passed through unchanged. -The build parallelism comes from ``CMAKE_BUILD_PARALLEL_LEVEL``, which the script passes on to ``ctest_build(PARALLEL_LEVEL ...)``. +The build parallelism comes from ``CMAKE_BUILD_PARALLEL_LEVEL``, which the script passes on to ``ctest_build(PARALLEL_LEVEL ...)``, and the generator from ``CMAKE_GENERATOR``. +Do not pass ``-G``: ``ctest_configure()`` appends its own generator argument after the options, which would silently override it, so the script rejects it instead. The script always builds the default target. Keep packaging steps such as ``--target pip_install`` outside of it: CTest scrapes the build log for compiler errors, and ``pip``'s setuptools warnings (``dist.py:318: UserWarning: ...``) match its error pattern, which would fail the job on a build that actually succeeded. diff --git a/Tools/CI/ctest_dashboard.cmake b/Tools/CI/ctest_dashboard.cmake index 283b3d3daf5..8eed25eb14b 100644 --- a/Tools/CI/ctest_dashboard.cmake +++ b/Tools/CI/ctest_dashboard.cmake @@ -6,67 +6,65 @@ # then running `ctest -D ExperimentalTest` produces a Test.xml only, which is # why our dashboard used to show test results and nothing else. # -# This script runs the configure and build steps via ctest_configure() and -# ctest_build(). The test step is deliberately left to the caller, so that CI -# keeps using the plain `ctest` command line it already has (including options -# such as `--no-tests=error`, which ctest_test() does not offer). Every step -# writes into the same dashboard tag created by ctest_start() here, so a single +# The command line route (`ctest -D ExperimentalConfigure`) cannot do this: it +# drives an already-generated build tree and needs a DartConfiguration.tcl, which +# include(CTest) only writes during a configure. It could therefore only time a +# *re*-configure, which for WarpX -- where the cold configure fetches AMReX, +# pyAMReX and PICSAR -- understates the real cost by minutes. Driving the +# configure from a dashboard client script, as here, is the documented way. +# +# The test step is deliberately left to the caller, so that CI keeps using the +# plain `ctest` command line it already has, including options such as +# `--no-tests=error` which ctest_test() does not offer. Every step writes into +# the same dashboard tag created by ctest_start() here, so a single # `ctest -D ExperimentalSubmit` at the end uploads all parts as one build. # # Do not invoke this file directly, use Tools/CI/ctest_dashboard.sh. # # Environment: -# CDASH_SOURCE_DIR top-level WarpX source directory (required) -# CDASH_BINARY_DIR build directory (required) -# CDASH_BUILD_NAME dashboard build name, e.g. "CPU-3D" (required) -# CDASH_SITE dashboard site name, e.g. "Azure" (required) -# CDASH_OPTIONS_FILE file with one CMake configure option per line -# CDASH_SUBMIT "ON" to submit right after the build (build-only jobs) -# CMAKE_GENERATOR CMake generator, unless given as -G in the options -# CMAKE_BUILD_PARALLEL_LEVEL build parallelism, passed on to ctest_build() +# CDASH_BUILD_DIR build directory, absolute (required) +# CDASH_CMAKE_ARGS configure options, shell-quoted by the wrapper +# CDASH_BUILD_NAME dashboard build name, e.g. "CPU-3D" (required) +# CDASH_SITE dashboard site name, e.g. "Azure" (required) +# CDASH_SUBMIT "ON" to submit right after the build (build-only jobs) +# CMAKE_GENERATOR CMake generator +# CMAKE_BUILD_PARALLEL_LEVEL build parallelism, passed to ctest_build() cmake_minimum_required(VERSION 3.25) -foreach(_required IN ITEMS CDASH_SOURCE_DIR CDASH_BINARY_DIR CDASH_BUILD_NAME CDASH_SITE) - if("$ENV{${_required}}" STREQUAL "") - message(FATAL_ERROR "ctest_dashboard: environment variable ${_required} is not set") +foreach(required IN ITEMS CDASH_BUILD_DIR CDASH_BUILD_NAME CDASH_SITE) + if("$ENV{${required}}" STREQUAL "") + message(FATAL_ERROR "ctest_dashboard: environment variable ${required} is not set") endif() endforeach() -set(CTEST_SOURCE_DIRECTORY "$ENV{CDASH_SOURCE_DIR}") -set(CTEST_BINARY_DIRECTORY "$ENV{CDASH_BINARY_DIR}") +get_filename_component(CTEST_SOURCE_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}/../.." ABSOLUTE) +set(CTEST_BINARY_DIRECTORY "$ENV{CDASH_BUILD_DIR}") set(CTEST_BUILD_NAME "$ENV{CDASH_BUILD_NAME}") set(CTEST_SITE "$ENV{CDASH_SITE}") -# Configure options are handed over one per line in a file, so that values -# containing spaces (-DCMAKE_CXX_FLAGS="-Werror -Wall") or semicolons -# (-DWarpX_DIMS="1;2") survive without any shell or CMake list quoting games. -set(configure_options "") -if(NOT "$ENV{CDASH_OPTIONS_FILE}" STREQUAL "") - # ENCODING: file(STRINGS) otherwise drops non-ASCII bytes silently - file(STRINGS "$ENV{CDASH_OPTIONS_FILE}" raw_options ENCODING UTF-8) - foreach(option IN LISTS raw_options) - # escape embedded ";" so that each line stays a single list element - string(REPLACE ";" "\;" option "${option}") - list(APPEND configure_options "${option}") - endforeach() -endif() +# The wrapper hands the options over shell-quoted; UNIX_COMMAND parses that back +# into one list element per argument and escapes any embedded ";" on the way, so +# values with spaces (-DCMAKE_CXX_FLAGS="-Werror -Wall") and semicolons +# (-DWarpX_DIMS="1;2") reach cmake exactly as CI wrote them. +separate_arguments(configure_options UNIX_COMMAND "$ENV{CDASH_CMAKE_ARGS}") -# ctest_configure() refuses to run without a generator, and appends -G after the -# options we pass. An explicit -G among them therefore has to be picked up here, -# or our own -G would silently override it. Both "-GNinja" and "-G;Ninja" occur. +# BUILDNAME and SITE are what include(CTest) writes into DartConfiguration.tcl, +# from where the caller's `ctest -D ExperimentalTest` picks them up for Test.xml. +# Without them that step would label Test.xml with the host name and a generic +# build name, and CDash would file it as a build separate from Configure.xml and +# Build.xml. +list(APPEND configure_options + "-DBUILDNAME=${CTEST_BUILD_NAME}" "-DSITE=${CTEST_SITE}") + +# ctest_configure() appends its own -G *after* these options, so one passed here +# would be silently overridden. Use the environment variable, which cmake reads +# natively, and say so rather than building the wrong thing. +if("${configure_options}" MATCHES "(^|;)-G") + message(FATAL_ERROR + "ctest_dashboard: pass the generator in CMAKE_GENERATOR, not as -G") +endif() set(CTEST_CMAKE_GENERATOR "$ENV{CMAKE_GENERATOR}") -set(next_is_generator FALSE) -foreach(option IN LISTS configure_options) - if(next_is_generator) - set(CTEST_CMAKE_GENERATOR "${option}") - set(next_is_generator FALSE) - elseif(option STREQUAL "-G") - set(next_is_generator TRUE) - elseif(option MATCHES "^-G(.+)$") - set(CTEST_CMAKE_GENERATOR "${CMAKE_MATCH_1}") - endif() -endforeach() if(CTEST_CMAKE_GENERATOR STREQUAL "") # same default as a plain `cmake` call on the platforms we submit from set(CTEST_CMAKE_GENERATOR "Unix Makefiles") @@ -84,7 +82,6 @@ endif() # ctest_start() reuses an existing Testing/TAG only while its date still matches # today's in UTC. A job that configures before and tests after midnight UTC # therefore splits into two dashboard builds; rare, and it heals on the next run. - ctest_start(Experimental) ctest_configure(OPTIONS "${configure_options}" RETURN_VALUE configure_rv) @@ -92,9 +89,9 @@ ctest_configure(OPTIONS "${configure_options}" RETURN_VALUE configure_rv) # A failed configure leaves nothing to build. set(build_rv 0) if(configure_rv EQUAL 0) - # Only ever the default target: CTest scrapes the build log for compiler - # errors, so a packaging target that shells out to pip would report its - # setuptools warnings as build errors. + # Only ever the default target: CTest finds build errors by matching regexes + # against the build log, so a packaging target that shells out to pip would + # have its setuptools warnings reported as build errors. ctest_build(${build_parallel_arg} RETURN_VALUE build_rv) endif() diff --git a/Tools/CI/ctest_dashboard.sh b/Tools/CI/ctest_dashboard.sh index f5ed88973b9..29575bcb4fd 100755 --- a/Tools/CI/ctest_dashboard.sh +++ b/Tools/CI/ctest_dashboard.sh @@ -7,22 +7,9 @@ # Tools/CI/ctest_dashboard.sh [cmake option ...] # # The CMake options are passed to the configure step unchanged; quote them as -# usual, values with spaces or semicolons are preserved. -# -# Environment: -# CDASH_BUILD_NAME dashboard build name, e.g. "CPU-3D" (required) -# CDASH_SITE dashboard site name, e.g. "Azure" (required) -# CDASH_SUBMIT "ON" submits to CDash right away; use this for build-only -# jobs. Jobs that also run tests should leave it "OFF" and -# submit once at the end, after their -# `ctest ... -D ExperimentalTest` step, with -# `ctest --test-dir -D ExperimentalSubmit`. -# A failed configure is always submitted from here, because -# that later command needs a DartConfiguration.tcl which -# does not exist yet when the configure dies early. -# -# The build parallelism comes from CMAKE_BUILD_PARALLEL_LEVEL, which is passed -# on to ctest_build(PARALLEL_LEVEL). +# usual, values with spaces or semicolons are preserved. Set the generator in +# CMAKE_GENERATOR rather than passing -G. See Tools/CI/ctest_dashboard.cmake +# for the environment this expects. set -o nounset -o errexit -o pipefail @@ -31,34 +18,22 @@ if [ $# -lt 1 ]; then exit 2 fi -build_dir="$1" -shift - script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -source_dir="$(cd -- "${script_dir}/../.." && pwd)" -mkdir -p "${build_dir}" -CDASH_BINARY_DIR="$(cd -- "${build_dir}" && pwd)" -export CDASH_BINARY_DIR +mkdir -p "$1" +CDASH_BUILD_DIR="$(cd -- "$1" && pwd)" +shift -export CDASH_SOURCE_DIR="${source_dir}" +# Hand the options over shell-quoted, for separate_arguments(UNIX_COMMAND) to +# parse back into exactly these arguments on the CMake side. +CDASH_CMAKE_ARGS="" +if [ $# -gt 0 ]; then + CDASH_CMAKE_ARGS="$(printf '%q ' "$@")" +fi + +export CDASH_BUILD_DIR CDASH_CMAKE_ARGS export CDASH_BUILD_NAME="${CDASH_BUILD_NAME:?CDASH_BUILD_NAME must be set}" export CDASH_SITE="${CDASH_SITE:?CDASH_SITE must be set}" export CDASH_SUBMIT="${CDASH_SUBMIT:-OFF}" -# hand the options over in a file: one per line, no quoting games needed -CDASH_OPTIONS_FILE="${CDASH_BINARY_DIR}/.cdash_configure_options" -export CDASH_OPTIONS_FILE -: > "${CDASH_OPTIONS_FILE}" -if [ $# -gt 0 ]; then - printf '%s\n' "$@" > "${CDASH_OPTIONS_FILE}" -fi -# BUILDNAME and SITE are what include(CTest) writes into DartConfiguration.tcl, -# from where a later `ctest -D ExperimentalTest` picks them up for Test.xml. -# Without them that step would label Test.xml with the host name and a generic -# build name, and CDash would file it as a build separate from Configure.xml and -# Build.xml. Appended last so they always match CTEST_BUILD_NAME/CTEST_SITE. -printf '%s\n' "-DBUILDNAME=${CDASH_BUILD_NAME}" "-DSITE=${CDASH_SITE}" \ - >> "${CDASH_OPTIONS_FILE}" - exec ctest -VV -S "${script_dir}/ctest_dashboard.cmake"