diff --git a/.github/actions/check-cmake-options/action.yml b/.github/actions/check-cmake-options/action.yml new file mode 100644 index 000000000000..92772fb78328 --- /dev/null +++ b/.github/actions/check-cmake-options/action.yml @@ -0,0 +1,45 @@ +name: "Check CMake options still exist upstream" +description: > + Fails if a GGML_*/LLAMA_* option this workflow passes was renamed or removed upstream. + CMake does not treat an unknown -D option as an error. + It warns and builds on without it, so a release can quietly lose a feature while staying green. + +inputs: + log: + description: "Path to the captured CMake configure output" + required: false + default: "cmake-configure.log" + +runs: + using: "composite" + steps: + - shell: bash + run: | + log='${{ inputs.log }}' + if [ ! -f "$log" ]; then + echo "::error::CMake configure log '$log' not found — the build step should have captured it." + exit 1 + fi + + # CMake indents the offenders under this header, between two blank lines. + unused=$(tr -d '\r' < "$log" | awk '/Manually-specified variables were not used/{c=1;next} c{if(NF==0){if(s)c=0;next} s=1;sub(/^[[:space:]]+/,"");print}' | sort -u) + if [ -z "$unused" ]; then + echo "Every CMake option this workflow passes is still recognized by upstream." + exit 0 + fi + + # Only GGML_*/LLAMA_* options change what we ship - fail. + # Toolchain, ROCm and HIP variables are read by other CMake packages + # and get listed as unused in healthy builds — warn, don't fail. + foreign=$(printf '%s\n' "$unused" | grep -vE '^(GGML|LLAMA)_' || true) + dropped=$(printf '%s\n' "$unused" | grep -E '^(GGML|LLAMA)_' || true) + + if [ -n "$foreign" ]; then + printf '::warning::CMake ignored -D%s — read by another CMake package, or no longer needed.\n' $foreign + fi + if [ -n "$dropped" ]; then + echo "::error::These CMake options no longer exist upstream (renamed or removed). The build silently dropped whatever those options enabled:" + printf '::error:: -D%s\n' $dropped + exit 1 + fi + echo "No llama.cpp/ggml options were dropped." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 47d70eeb2b29..9205882d2e21 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,6 +111,7 @@ jobs: - name: Build with native CMake HIP support id: cmake_build run: | + set -o pipefail cmake -B build -S . \ -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \ -DCMAKE_BUILD_TYPE=Release \ @@ -124,9 +125,17 @@ jobs: -DHIP_PLATFORM=amd \ -DGGML_HIP_ROCWMMA_FATTN=ON \ -DGGML_OPENMP=ON \ - ${{ env.CMAKE_ARGS }} + ${{ env.CMAKE_ARGS }} 2>&1 | tee cmake-configure.log cmake --build build --config Release -j $(nproc) + - name: Check out CI helpers + uses: actions/checkout@v6 + with: + path: lemonade-ci + + - name: Verify CMake options still exist upstream + uses: ./lemonade-ci/.github/actions/check-cmake-options + - name: Determine tag name id: tag uses: lemonade-sdk/llama.cpp/.github/actions/get-tag-name@lemonade @@ -230,6 +239,7 @@ jobs: run: | cmake_arch="${{ matrix.sm }}" cmake_arch="${cmake_arch#sm_}" + set -o pipefail cmake -B build -S . \ -DGGML_CUDA=ON \ -DCMAKE_CUDA_ARCHITECTURES="${cmake_arch}" \ @@ -240,9 +250,17 @@ jobs: -DGGML_STATIC=OFF \ -DLLAMA_BUILD_BORINGSSL=ON \ -DCMAKE_BUILD_TYPE=Release \ - ${{ env.CMAKE_ARGS }} + ${{ env.CMAKE_ARGS }} 2>&1 | tee cmake-configure.log cmake --build build --config Release -j $(nproc) + - name: Check out CI helpers + uses: actions/checkout@v6 + with: + path: lemonade-ci + + - name: Verify CMake options still exist upstream + uses: ./lemonade-ci/.github/actions/check-cmake-options + - name: Bundle CUDA runtime libraries run: | cuda_lib=/usr/local/cuda/lib64 @@ -412,6 +430,7 @@ jobs: run: | cmake_arch="${{ matrix.sm }}" cmake_arch="${cmake_arch#sm_}" + set -o pipefail cmake -B build -S . \ -DGGML_CUDA=ON \ -DCMAKE_CUDA_ARCHITECTURES="${cmake_arch}" \ @@ -422,9 +441,17 @@ jobs: -DGGML_STATIC=OFF \ -DLLAMA_BUILD_BORINGSSL=ON \ -DCMAKE_BUILD_TYPE=Release \ - ${{ env.CMAKE_ARGS }} + ${{ env.CMAKE_ARGS }} 2>&1 | tee cmake-configure.log cmake --build build --config Release -j $(nproc) + - name: Check out CI helpers + uses: actions/checkout@v6 + with: + path: lemonade-ci + + - name: Verify CMake options still exist upstream + uses: ./lemonade-ci/.github/actions/check-cmake-options + - name: Bundle CUDA runtime libraries run: | cuda_lib=/usr/local/cuda/lib64 @@ -522,12 +549,20 @@ jobs: path: llama-ubuntu-cuda-${{ matrix.sm }}-arm64.tar.xz name: llama-ubuntu-cuda-${{ matrix.sm }}-arm64.tar.xz - ubuntu-22-openvino: - runs-on: ubuntu-22.04 + ubuntu-openvino: + # Two Ubuntu baselines: + # 24.04 is what upstream builds and validates the OpenVINO backend on, + # 22.04 keeps a glibc 2.35 binary for users on older distros. + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + ubuntu: "22" + - os: ubuntu-24.04 + ubuntu: "24" - env: - OPENVINO_VERSION_MAJOR: "2026.0" - OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886" + runs-on: ${{ matrix.os }} steps: - name: Clone @@ -537,14 +572,47 @@ jobs: fetch-depth: 0 repository: 'ggml-org/llama.cpp' + # The ggml-openvino backend is written against a specific OpenVINO release. + # So a hand-maintained pin here silently breaks the build the moment upstream moves. + # Read the version out of the upstream workflow, so this job always builds with the toolkit upstream built with. + # If upstream ever restructures that file this step fails rather than falling back to a stale version. + - name: Resolve OpenVINO version from upstream + id: openvino_version + run: | + upstream_workflow=.github/workflows/release.yml + major=$(grep -oP 'OPENVINO_VERSION_MAJOR:\s*"\K[^"]+' "$upstream_workflow" | sort -u || true) + full=$(grep -oP 'OPENVINO_VERSION_FULL:\s*"\K[^"]+' "$upstream_workflow" | sort -u || true) + + if [ "$(printf '%s' "$major" | grep -c .)" -ne 1 ] || [ "$(printf '%s' "$full" | grep -c .)" -ne 1 ]; then + echo "::error::Could not resolve a single OpenVINO version from upstream $upstream_workflow." + echo "::error::OPENVINO_VERSION_MAJOR candidates: ${major:-}" + echo "::error::OPENVINO_VERSION_FULL candidates: ${full:-}" + echo "::error::Upstream probably renamed these variables or split the OpenVINO jobs across files — update this step." + exit 1 + fi + + # The toolkit URL is /linux/openvino_toolkit__, so a major that is not a prefix of full means we would download a 404. + case "$full" in + "$major".*) ;; + *) + echo "::error::OpenVINO version mismatch upstream: full '$full' is not a build of major '$major'." + exit 1 + ;; + esac + + echo "Building against OpenVINO $full (as pinned by upstream)" + echo "OPENVINO_VERSION_MAJOR=$major" >> "$GITHUB_ENV" + echo "OPENVINO_VERSION_FULL=$full" >> "$GITHUB_ENV" + - name: ccache uses: ggml-org/ccache-action@v1.2.21 with: - key: ubuntu-openvino + key: ubuntu-${{ matrix.ubuntu }}-openvino evict-old-files: 1d - name: Dependencies run: | + sudo apt-get update sudo apt-get install -y build-essential cmake ninja-build patchelf \ python3-pip libtbb12 \ ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd @@ -555,6 +623,7 @@ jobs: path: ${{ github.workspace }}/openvino version_major: ${{ env.OPENVINO_VERSION_MAJOR }} version_full: ${{ env.OPENVINO_VERSION_FULL }} + ubuntu_version: ubuntu${{ matrix.ubuntu }} - name: Install OpenVINO dependencies run: | @@ -564,6 +633,7 @@ jobs: - name: Build run: | source ${{ github.workspace }}/openvino/setupvars.sh + set -o pipefail cmake -B build -S . \ -DGGML_OPENVINO=ON \ -DBUILD_SHARED_LIBS=ON \ @@ -573,9 +643,17 @@ jobs: -DGGML_STATIC=OFF \ -DLLAMA_BUILD_BORINGSSL=ON \ -DCMAKE_BUILD_TYPE=Release \ - ${{ env.CMAKE_ARGS }} + ${{ env.CMAKE_ARGS }} 2>&1 | tee cmake-configure.log cmake --build build --config Release -j $(nproc) + - name: Check out CI helpers + uses: actions/checkout@v6 + with: + path: lemonade-ci + + - name: Verify CMake options still exist upstream + uses: ./lemonade-ci/.github/actions/check-cmake-options + - name: Bundle OpenVINO runtime libraries run: | OV_LIBDIR="${{ github.workspace }}/openvino/runtime/lib/intel64" @@ -624,6 +702,7 @@ jobs: fi "$smoke_bin" --version >/dev/null + problems=0 for f in build/bin/*; do [ -f "$f" ] && ! [ -L "$f" ] || continue if ! file "$f" | grep -q 'ELF'; then @@ -633,18 +712,25 @@ jobs: rpath=$(patchelf --print-rpath "$f") if [ "$rpath" != '$ORIGIN' ]; then echo "::error::Unexpected RPATH '$rpath' for $f" - exit 1 + problems=$((problems + 1)) fi - # libOpenCL is optional (Intel GPU acceleration); skip it - missing=$(ldd "$f" | awk '/=> not found/ && $1 != "libOpenCL.so.1" { print }') + # Both of these come from the user's driver stack: libOpenCL from the GPU driver, libze_loader (level-zero) + # OpenVINO only loads the device plugin from the Intel NPU driver. + # that needs them when the matching driver is installed. + missing=$(ldd "$f" | awk '/=> not found/ && $1 != "libOpenCL.so.1" && $1 != "libze_loader.so.1" { print }') if [ -n "$missing" ]; then echo "::error::Unresolved runtime dependencies for $f" echo "$missing" - exit 1 + problems=$((problems + 1)) fi done + if [ "$problems" -ne 0 ]; then + echo "::error::OpenVINO package validation found $problems problem(s) — see the errors above." + exit 1 + fi + - name: Determine tag name id: tag uses: lemonade-sdk/llama.cpp/.github/actions/get-tag-name@lemonade @@ -652,15 +738,15 @@ jobs: - name: Pack artifacts run: | cp LICENSE ./build/bin/ - tar -czvf llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz \ + tar -czvf llama-bin-ubuntu-${{ matrix.ubuntu }}-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz \ --transform "s,./,llama-${{ steps.tag.outputs.name }}/," \ -C ./build/bin . - name: Upload artifacts uses: actions/upload-artifact@v6 with: - path: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz - name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz + path: llama-bin-ubuntu-${{ matrix.ubuntu }}-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz + name: llama-bin-ubuntu-${{ matrix.ubuntu }}-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz windows-cpu: runs-on: windows-2025 @@ -700,9 +786,20 @@ jobs: -DGGML_BACKEND_DL=ON ^ -DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} ^ -DGGML_OPENMP=ON ^ - ${{ env.CMAKE_ARGS }} + ${{ env.CMAKE_ARGS }} > cmake-configure.log 2>&1 + set CMAKE_STATUS=%ERRORLEVEL% + type cmake-configure.log + if not "%CMAKE_STATUS%"=="0" exit /b %CMAKE_STATUS% cmake --build build --config Release + - name: Check out CI helpers + uses: actions/checkout@v6 + with: + path: lemonade-ci + + - name: Verify CMake options still exist upstream + uses: ./lemonade-ci/.github/actions/check-cmake-options + - name: Pack artifacts id: pack_artifacts shell: pwsh @@ -835,6 +932,7 @@ jobs: run: | mkdir build cd build + $ErrorActionPreference = 'Continue' cmake .. ` -G "Unix Makefiles" ` -DCMAKE_PREFIX_PATH="${env:HIP_PATH}" ` @@ -850,9 +948,22 @@ jobs: -DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" ` -DHIP_PATH="${env:HIP_PATH}" ` -DGGML_HIP_ROCWMMA_FATTN=ON ` - -DAMDGPU_TARGETS="${{ matrix.gpu_targets }}" + -DAMDGPU_TARGETS="${{ matrix.gpu_targets }}" 2>&1 | Tee-Object -FilePath cmake-configure.log + $cmakeStatus = $LASTEXITCODE + $ErrorActionPreference = 'Stop' + if ($cmakeStatus -ne 0) { exit $cmakeStatus } cmake --build . --config Release --parallel ${env:NUMBER_OF_PROCESSORS} + - name: Check out CI helpers + uses: actions/checkout@v6 + with: + path: lemonade-ci + + - name: Verify CMake options still exist upstream + uses: ./lemonade-ci/.github/actions/check-cmake-options + with: + log: build/cmake-configure.log + - name: Verify HIP backend was built run: | $hipDll = Get-ChildItem -Path build\bin -Filter "ggml-hip*.dll" -ErrorAction SilentlyContinue @@ -934,9 +1045,20 @@ jobs: -DGGML_NATIVE=OFF ^ -DGGML_BACKEND_DL=ON ^ -DLLAMA_BUILD_BORINGSSL=ON ^ - ${{ env.CMAKE_ARGS }} + ${{ env.CMAKE_ARGS }} > cmake-configure.log 2>&1 + set CMAKE_STATUS=%ERRORLEVEL% + type cmake-configure.log + if not "%CMAKE_STATUS%"=="0" exit /b %CMAKE_STATUS% cmake --build build --config Release + - name: Check out CI helpers + uses: actions/checkout@v6 + with: + path: lemonade-ci + + - name: Verify CMake options still exist upstream + uses: ./lemonade-ci/.github/actions/check-cmake-options + - name: Download CPU backend artifact uses: actions/download-artifact@v7 with: @@ -1047,7 +1169,7 @@ jobs: - ubuntu-24-rocm - ubuntu-22-cuda - ubuntu-22-cuda-arm64 - - ubuntu-22-openvino + - ubuntu-openvino steps: - name: Clone @@ -1057,6 +1179,35 @@ jobs: fetch-depth: 0 repository: 'ggml-org/llama.cpp' + # Surface the difference between upstream's shared release flags and ours on every run (workflow-level CMAKE_ARGS only). + # Per-job flags are deliberately (packaging, backend selection) not compared against upstream's. + # Options upstream *removes* are caught, and failed on, by the "Verify CMake options still exist upstream" step in each build job. + - name: Report upstream CMake arg drift + run: | + # Two-space indent anchors this to the workflow-level env block, so a job-level CMAKE_ARGS override upstream can never be picked up instead. + theirs_raw=$(grep -oP '^ CMAKE_ARGS:\s*"\K[^"]+' .github/workflows/release.yml || true) + if [ "$(printf '%s' "$theirs_raw" | grep -c .)" -ne 1 ]; then + echo "::warning::Expected exactly one workflow-level CMAKE_ARGS in upstream's release workflow, found $(printf '%s' "$theirs_raw" | grep -c .) — upstream may have restructured it. Skipping the drift comparison." + exit 0 + fi + + ours=$(printf '%s\n' ${{ env.CMAKE_ARGS }} | sort -u) + theirs=$(printf '%s\n' $theirs_raw | sort -u) + + added=$(comm -13 <(printf '%s\n' "$ours") <(printf '%s\n' "$theirs")) + removed=$(comm -23 <(printf '%s\n' "$ours") <(printf '%s\n' "$theirs")) + + if [ -z "$added" ] && [ -z "$removed" ]; then + echo "Shared CMake args match upstream exactly." + exit 0 + fi + if [ -n "$added" ]; then + printf '::warning::Upstream release builds pass %s but this workflow does not. Consider whether these artifacts should too.\n' $added + fi + if [ -n "$removed" ]; then + printf '::warning::This workflow passes %s but upstream release builds do not. Consider whether it is wanted.\n' $removed + fi + - name: Determine tag name id: tag uses: lemonade-sdk/llama.cpp/.github/actions/get-tag-name@lemonade @@ -1196,7 +1347,9 @@ jobs: declare -A family_glob=( [cuda]="release/llama-${TAG}-*cuda-sm_*" [rocm]="release/llama-${TAG}-bin-*rocm-*" - [openvino]="release/llama-${TAG}-bin-*openvino-*" + # The two Ubuntu baselines are separate families targetting different glibc versions. + [openvino-22]="release/llama-${TAG}-bin-ubuntu-22-openvino-*" + [openvino-24]="release/llama-${TAG}-bin-ubuntu-24-openvino-*" [cpu]="release/llama-${TAG}-bin-win-cpu-*" ) @@ -1210,11 +1363,11 @@ jobs: # Each backend family is checked independently, but must be # published as a complete, internally-consistent set across every - # platform/variant it covers (e.g. all 7 CUDA sm_* builds for + # platform/variant it covers (e.g. all 8 CUDA sm_* builds for # ubuntu x64/arm64 and windows) — never a partial mix. A gap in one # family drops that whole family from the release; other families # are unaffected. - for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120; do + for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120 sm_121; do check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-x64.tar.xz" check_glob cuda "release/llama-${TAG}-ubuntu-cuda-${sm}-arm64.tar.xz" check_glob cuda "release/llama-${TAG}-windows-cuda-${sm}-x64.7z" @@ -1222,7 +1375,8 @@ jobs: check_glob rocm "release/llama-${TAG}-bin-ubuntu-rocm-*-x64.tar.gz" check_glob rocm "release/llama-${TAG}-bin-win-rocm-*-x64.zip" - check_glob openvino "release/llama-${TAG}-bin-ubuntu-openvino-*-x64.tar.gz" + check_glob openvino-22 "release/llama-${TAG}-bin-ubuntu-22-openvino-*-x64.tar.gz" + check_glob openvino-24 "release/llama-${TAG}-bin-ubuntu-24-openvino-*-x64.tar.gz" check_glob cpu "release/llama-${TAG}-bin-win-cpu-x64.zip" missing=0 @@ -1266,6 +1420,20 @@ jobs: script: | const tag = process.env.TAG_NAME; const { owner, repo } = context.repo; + + // The OpenVINO version is resolved from upstream at build time, so it is not known here. + // Derive the download links from the files that were actually staged for this release + // instead of hardcoding a version that would go stale (or 404) the next time it moves. + const fs = require('fs'); + const openvinoLinks = (fs.existsSync('release') ? fs.readdirSync('release') : []) + .filter(f => f.includes('-openvino-') && f.endsWith('.tar.gz')) + .sort() + .map(f => { + const m = f.match(/-bin-ubuntu-(\d+)-openvino-(.+)-x64\.tar\.gz$/); + const label = m ? `Ubuntu ${m[1]}.04 x64 (OpenVINO ${m[2]})` : f; + return `- [${label}](https://github.com/${owner}/${repo}/releases/download/${tag}/${f})`; + }); + let release; try { release = (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data; @@ -1285,7 +1453,7 @@ jobs: `- [Ubuntu x64 (ROCm 10.1)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-ubuntu-rocm-10.1-x64.tar.gz)`, '- Ubuntu x64 (CUDA): `llama-' + tag + '-ubuntu-cuda-sm_XX-x64.tar.xz` (replace XX with your GPU compute capability)', '- Ubuntu arm64 (CUDA): `llama-' + tag + '-ubuntu-cuda-sm_XX-arm64.tar.xz` (replace XX with your GPU compute capability)', - `- [Ubuntu x64 (OpenVINO 2026.0)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-ubuntu-openvino-2026.0-x64.tar.gz)`, + ...openvinoLinks, '', '**Windows:**', `- [Windows x64 (ROCm 7.14)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-win-rocm-7.14-x64.zip)`,