From d3200190c1fc5d76d3c27e50204eb99b9155cd21 Mon Sep 17 00:00:00 2001 From: Kevin Blackburn-Matzen Date: Sat, 18 Jul 2026 16:23:28 -0700 Subject: [PATCH] Fail CI when a test fixture fails to generate Every fixture is produced by ffmpeg at configure time and the tests that use them are #ifdef-guarded, so a fixture that fails to generate removes test cases from the suite without failing anything. That is the same shape of silent gap as #55, where CI ran zero tests and stayed green: a pass that quietly means less than it appears to. CI installs all the dependencies these fixtures need, so on CI a disabled fixture is a real problem rather than graceful degradation. The configure output is now captured and the build fails if any fixture reports DISABLED, listing which one. The HDR fixture's message is reworded to the same "NAME fixture: enabled/DISABLED" form as the others so one grep covers all of them. Verified both directions: all three report enabled locally and on both CI platforms, and the guard does trip on a synthesised DISABLED line. Refs #58 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 18 +++++++++++++++++- CMakeLists.txt | 3 ++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea2ae9a..221ff22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,12 +53,28 @@ jobs: - name: Configure run: | + set -o pipefail cmake -B build \ -DFRAMEWRIGHT_BUILD_TESTS=ON \ -DFRAMEWRIGHT_BUILD_EXAMPLES=ON \ -DFRAMEWRIGHT_BUILD_PYTHON=ON \ -Dpybind11_DIR=$(python3 -c "import pybind11; print(pybind11.get_cmake_dir())") \ - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=Release 2>&1 | tee configure.log + + # Every test fixture is generated by ffmpeg at configure time, and the + # test cases that use them are #ifdef-guarded. A fixture that fails to + # generate therefore removes tests from the suite without failing + # anything -- the same shape of silent gap as #55, where CI ran zero + # tests and stayed green. All dependencies are installed above, so on CI + # a disabled fixture is a real problem, not a graceful degradation. + - name: Require all fixtures + run: | + if grep -q "fixture: DISABLED" configure.log; then + grep "fixture: DISABLED" configure.log + echo "::error::a test fixture failed to generate; the suite would silently shrink" + exit 1 + fi + grep "fixture: enabled" configure.log - name: Build run: cmake --build build -j$(nproc 2>/dev/null || sysctl -n hw.ncpu) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ddf0dd..5269253 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,9 +204,10 @@ if(FRAMEWRIGHT_BUILD_TESTS) ) if(HDR_FIXTURE_RESULT EQUAL 0) set(HAVE_HDR_FIXTURE TRUE) + message(STATUS "HDR fixture: enabled") else() set(HAVE_HDR_FIXTURE FALSE) - message(STATUS "libx265 not available, HDR reader test fixture skipped") + message(STATUS "HDR fixture: DISABLED, libx265 not available") endif() else() message(WARNING "ffmpeg not found, test fixtures will not be generated")