release.yml

name: Build / Release

run-name: ${{ inputs.publish_release && format('Release {0}', inputs.tag_name) || format('Build {0} Preview', inputs.source_ref) }}

on:
  workflow_dispatch:
    inputs:
      source_ref:
        description: "Branch or tag name that requested the build"
        required: true
        type: string
      source_sha:
        description: "Exact commit SHA to build"
        required: true
        type: string
      tag_name:
        description: "Release/package version name, for example v0.4.0"
        required: true
        type: string
      publish_release:
        description: "Publish a stable release from its v* tag"
        required: true
        default: false
        type: boolean

concurrency:
  group: ${{ github.workflow }}-${{ inputs.source_ref }}-${{ inputs.publish_release && 'stable' || 'preview' }}
  cancel-in-progress: ${{ !inputs.publish_release }}

permissions:
  actions: write
  contents: read

env:
  CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_IS_DEV=OFF -DLLAMA_BUILD_UI=ON -DLLAMA_USE_PREBUILT_UI=OFF -DGGML_RPC=ON"

jobs:
  release-meta:
    name: Publication metadata
    runs-on: ubuntu-24.04
    outputs:
      source_ref: ${{ steps.meta.outputs.source_ref }}
      source_sha: ${{ steps.meta.outputs.source_sha }}
      tag_name: ${{ steps.meta.outputs.tag_name }}
      version_name: ${{ steps.meta.outputs.version_name }}
      release_title: ${{ steps.meta.outputs.release_title }}
      preview: ${{ steps.meta.outputs.preview }}
      stable_release: ${{ steps.meta.outputs.stable_release }}
      build_packages: ${{ steps.meta.outputs.build_packages }}
      publish_docker: ${{ steps.meta.outputs.publish_docker }}
      cache_channel: ${{ steps.meta.outputs.cache_channel }}
      cache_parent_channel: ${{ steps.meta.outputs.cache_parent_channel }}
      image_repo: ${{ steps.meta.outputs.image_repo }}
      short_sha: ${{ steps.meta.outputs.short_sha }}
    steps:
      - name: Prune stale release caches
        uses: actions/github-script@v8
        with:
          script: |
            const { owner, repo } = context.repo;
            const liveBranches = [];
            const caches = [];

            for (let page = 1; ; page += 1) {
              const { data } = await github.rest.repos.listBranches({
                owner,
                repo,
                per_page: 100,
                page,
              });
              liveBranches.push(...data
                .map((branch) => branch.name)
                .filter((name) => /^v[0-9][0-9A-Za-z._-]*$/.test(name)));
              if (data.length < 100) {
                break;
              }
            }

            for (let page = 1; ; page += 1) {
              const { data } = await github.rest.actions.getActionsCacheList({
                owner,
                repo,
                per_page: 100,
                page,
              });
              const pageCaches = data.actions_caches ?? [];
              caches.push(...pageCaches);
              if (pageCaches.length < 100) {
                break;
              }
            }

            let deleted = 0;
            for (const cache of caches) {
              if (!cache.key.startsWith("release-")) {
                continue;
              }
              const liveChannel = liveBranches.some((branch) => {
                const prefix = `release-${branch}-`;
                if (!cache.key.startsWith(prefix)) {
                  return false;
                }
                return /^(macos|ubuntu|windows)-/.test(cache.key.slice(prefix.length));
              });
              if (cache.ref === "refs/heads/main" && liveChannel) {
                continue;
              }
              try {
                await github.rest.actions.deleteActionsCacheById({
                  owner,
                  repo,
                  cache_id: cache.id,
                });
                deleted += 1;
                core.info(`Deleted stale release cache ${cache.key} on ${cache.ref}.`);
              } catch (error) {
                if (error.status !== 404) {
                  throw error;
                }
              }
            }
            core.info(`Kept release caches for live branches: ${liveBranches.join(", ") || "none"}.`);
            core.info(`Deleted ${deleted} stale or incorrectly scoped release cache entries.`);

      - name: Clone
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Resolve release metadata
        id: meta
        shell: bash
        run: |
          set -euo pipefail

          build_packages="true"
          publish_docker="true"

          if [[ "${EVENT_NAME}" != "workflow_dispatch" ]]; then
            echo "Unsupported event ${EVENT_NAME}; builds must use main-owned workflow_dispatch." >&2
            exit 1
          fi
          if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then
            echo "Build workflow_dispatch must run on main, not ${GITHUB_REF}." >&2
            exit 1
          fi

          source_ref="${INPUT_SOURCE_REF}"
          source_sha="${INPUT_SOURCE_SHA}"
          tag_name="${INPUT_TAG_NAME}"

          if [[ "${INPUT_PUBLISH_RELEASE}" == "true" ]]; then
            version_name="${tag_name}"
            release_title="${tag_name}"
            preview="false"
            stable_release="true"
          else
            version_name="${source_ref}"
            release_title="${source_ref} Preview"
            preview="true"
            stable_release="false"
          fi

          if [[ ! "${version_name}" =~ ^v[0-9][0-9A-Za-z._-]*$ ]]; then
            echo "Version name ${version_name} is not safe for release assets and Docker tags." >&2
            exit 1
          fi

          if [[ -z "${source_sha}" || ! "${source_sha}" =~ ^[0-9a-f]{40}$ ]]; then
            echo "source_sha must be a full 40-character commit SHA." >&2
            exit 1
          fi
          if ! git cat-file -e "${source_sha}^{commit}"; then
            git fetch origin +refs/heads/"${source_ref}":refs/remotes/origin/"${source_ref}" --no-tags || true
            git fetch origin +refs/tags/"${source_ref}":refs/tags/"${source_ref}" || true
          fi
          if ! git cat-file -e "${source_sha}^{commit}"; then
            git fetch origin "${source_sha}" --no-tags
          fi
          git cat-file -e "${source_sha}^{commit}"

          cmake_lists="$(git show "${source_sha}:CMakeLists.txt")"
          version_major="$(sed -nE 's/^set\(LLAMA_VERSION_MAJOR[[:space:]]+([0-9]+)\)[[:space:]]*$/\1/p' <<< "${cmake_lists}")"
          version_minor="$(sed -nE 's/^set\(LLAMA_VERSION_MINOR[[:space:]]+([0-9]+)\)[[:space:]]*$/\1/p' <<< "${cmake_lists}")"
          version_patch="$(sed -nE 's/^set\(LLAMA_VERSION_PATCH[[:space:]]+([0-9]+)\)[[:space:]]*$/\1/p' <<< "${cmake_lists}")"
          if [[ -z "${version_major}" || -z "${version_minor}" || -z "${version_patch}" ]]; then
            echo "Could not read the release version from CMakeLists.txt at ${source_sha}." >&2
            exit 1
          fi
          source_version="${version_major}.${version_minor}.${version_patch}"
          if [[ "${version_name#v}" != "${source_version}" ]]; then
            echo "Requested version ${version_name} does not match source version v${source_version}." >&2
            exit 1
          fi

          if [[ "${preview}" == "true" ]]; then
            if [[ "${source_ref}" != v* || "${tag_name}" != "preview-${source_ref}" ]]; then
              echo "Invalid preview metadata: source_ref=${source_ref}, tag_name=${tag_name}." >&2
              exit 1
            fi
            git fetch origin +refs/heads/"${source_ref}":refs/remotes/origin/"${source_ref}" --no-tags
            branch_commit="$(git rev-parse refs/remotes/origin/"${source_ref}")"
            if [[ "${branch_commit}" != "${source_sha}" ]]; then
              echo "Branch ${source_ref} points to ${branch_commit}, not requested source ${source_sha}." >&2
              exit 1
            fi
          else
            if [[ "${tag_name}" != v* ]]; then
              echo "Release publishing requires a v* tag_name." >&2
              exit 1
            fi
            if [[ "${source_ref}" != "${tag_name}" ]]; then
              echo "Stable release source_ref ${source_ref} must match tag_name ${tag_name}." >&2
              exit 1
            fi
            git fetch origin +refs/tags/"${tag_name}":refs/tags/"${tag_name}"
            tag_commit="$(git rev-list -n 1 "${tag_name}")"
            if [[ "${tag_commit}" != "${source_sha}" ]]; then
              echo "Tag ${tag_name} points to ${tag_commit}, not requested source ${source_sha}." >&2
              exit 1
            fi
            git fetch origin +refs/heads/main:refs/remotes/origin/main --no-tags
            if ! git merge-base --is-ancestor "${source_sha}" refs/remotes/origin/main; then
              echo "Tag ${tag_name} is not reachable from origin/main; refusing to publish." >&2
              exit 1
            fi
            git fetch origin +refs/heads/"${source_ref}":refs/remotes/origin/"${source_ref}" --no-tags
            branch_commit="$(git rev-parse refs/remotes/origin/"${source_ref}")"
            if ! git merge-base --is-ancestor "${branch_commit}" "${source_sha}"; then
              echo "Release branch ${source_ref} at ${branch_commit} is not an ancestor of tag ${source_sha}." >&2
              exit 1
            fi
          fi

          cache_channel="${source_ref}"
          cache_parent_channel="$(
            python3 .github/scripts/resolve-release-cache-parent.py \
              --source-ref "${source_ref}" \
              --source-sha "${source_sha}"
          )"

          owner="$(printf '%s' "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')"
          repo="$(printf '%s' "${GITHUB_REPOSITORY#*/}" | tr '[:upper:]' '[:lower:]')"
          image_repo="ghcr.io/${owner}/${repo}"

          echo "source_ref=${source_ref}" >> "${GITHUB_OUTPUT}"
          echo "source_sha=${source_sha}" >> "${GITHUB_OUTPUT}"
          echo "tag_name=${tag_name}" >> "${GITHUB_OUTPUT}"
          echo "version_name=${version_name}" >> "${GITHUB_OUTPUT}"
          echo "release_title=${release_title}" >> "${GITHUB_OUTPUT}"
          echo "preview=${preview}" >> "${GITHUB_OUTPUT}"
          echo "stable_release=${stable_release}" >> "${GITHUB_OUTPUT}"
          echo "build_packages=${build_packages}" >> "${GITHUB_OUTPUT}"
          echo "publish_docker=${publish_docker}" >> "${GITHUB_OUTPUT}"
          echo "cache_channel=${cache_channel}" >> "${GITHUB_OUTPUT}"
          echo "cache_parent_channel=${cache_parent_channel}" >> "${GITHUB_OUTPUT}"
          echo "image_repo=${image_repo}" >> "${GITHUB_OUTPUT}"
          short_sha="${source_sha::12}"
          echo "short_sha=${short_sha}" >> "${GITHUB_OUTPUT}"
          echo "Using source ref: ${source_ref}"
          echo "Using source SHA: ${source_sha}"
          echo "Using package version: ${tag_name}"
          echo "Release title: ${release_title}"
          echo "Preview: ${preview}"
          echo "Build packages: ${build_packages}"
          echo "Publish Docker images: ${publish_docker}"
          echo "Cache parent channel: ${cache_parent_channel:-none}"
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUT_SOURCE_REF: ${{ inputs.source_ref }}
          INPUT_SOURCE_SHA: ${{ inputs.source_sha }}
          INPUT_TAG_NAME: ${{ inputs.tag_name }}
          INPUT_PUBLISH_RELEASE: ${{ inputs.publish_release }}

  macos-arm64:
    name: macOS arm64 Metal
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: macos-14
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-macos-arm64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-macos-arm64

      - name: Build
        id: build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DGGML_METAL_EMBED_LIBRARY=ON \
            -DCMAKE_INSTALL_RPATH='@loader_path' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            -DLLAMA_BUILD_BORINGSSL=ON \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            ${{ env.CMAKE_ARGS }}
          if python3 .github/scripts/cache-build-deadline.py -- cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)"; then
            :
          else
            result=$?
            if [ "$result" -eq 75 ]; then exit 0; fi
            exit "$result"
          fi
          test -s build/tools/ui/dist/index.html

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-macos-arm64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Smoke package binaries
        run: |
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-macos-arm64.tar.gz" \
            -s ",^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-macos-arm64.tar.gz
          name: beellama-bin-macos-arm64.tar.gz

  ubuntu-cpu:
    name: Ubuntu ${{ matrix.build }} CPU
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - build: x64
            os: ubuntu-22.04
          - build: arm64
            os: ubuntu-24.04-arm
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-${{ matrix.os }}-cpu
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-${{ matrix.os }}-cpu

      - name: Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential libssl-dev

      - name: Toolchain workaround
        if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
        run: |
          sudo apt-get install -y gcc-14 g++-14
          echo "CC=gcc-14" >> "${GITHUB_ENV}"
          echo "CXX=g++-14" >> "${GITHUB_ENV}"

      - name: Build
        id: build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            ${{ env.CMAKE_ARGS }}
          if python3 .github/scripts/cache-build-deadline.py -- cmake --build build --config Release -j "$(nproc)"; then
            :
          else
            result=$?
            if [ "$result" -eq 75 ]; then exit 0; fi
            exit "$result"
          fi
          test -s build/tools/ui/dist/index.html

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-${{ matrix.os }}-cpu
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Smoke package binaries
        run: |
          file ./build/bin/llama-server
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-${{ matrix.build }}.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-${{ matrix.build }}.tar.gz
          name: beellama-bin-ubuntu-${{ matrix.build }}.tar.gz

  ubuntu-vulkan:
    name: Ubuntu x64 Vulkan
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-22.04
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-vulkan-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-ubuntu-vulkan-x64

      - name: Dependencies
        run: |
          wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -
          sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
          sudo apt-get update -y
          sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev

      - name: Build
        id: build
        run: |
          cmake -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGGML_VULKAN=ON \
            -DLLAMA_FATAL_WARNINGS=ON \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            ${{ env.CMAKE_ARGS }}
          if python3 .github/scripts/cache-build-deadline.py -- cmake --build build --config Release -j "$(nproc)"; then
            :
          else
            result=$?
            if [ "$result" -eq 75 ]; then exit 0; fi
            exit "$result"
          fi
          test -s build/tools/ui/dist/index.html

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-vulkan-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Smoke package binaries
        run: |
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-vulkan-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-vulkan-x64.tar.gz
          name: beellama-bin-ubuntu-vulkan-x64.tar.gz

  ubuntu-rocm:
    name: Ubuntu x64 ROCm
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-22.04
    env:
      ROCM_VERSION: "7.2.1"
      GPU_TARGETS: "gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx1150;gfx1200;gfx1201"
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Free up disk space
        uses: ggml-org/free-disk-space@v1.3.1
        with:
          tool-cache: true

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-rocm-${{ env.ROCM_VERSION }}-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-ubuntu-rocm-${{ env.ROCM_VERSION }}-x64

      - name: Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential git cmake wget libssl-dev

      - name: Setup ROCm
        run: |
          sudo mkdir --parents --mode=0755 /etc/apt/keyrings
          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
            gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
          sudo tee /etc/apt/sources.list.d/rocm.list << EOF
          deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} jammy main
          EOF
          sudo tee /etc/apt/preferences.d/rocm-pin-600 << EOF
          Package: *
          Pin: release o=repo.radeon.com
          Pin-Priority: 600
          EOF
          sudo apt-get update
          sudo apt-get install -y rocm-hip-sdk

      - name: Build
        id: build
        run: |
          cmake -B build -S . \
            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
            -DCMAKE_BUILD_TYPE=Release \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DGPU_TARGETS="${GPU_TARGETS}" \
            -DGGML_HIP=ON \
            -DGGML_CUDA_KVARN=ON \
            -DGGML_CUDA_FA=ON \
            -DHIP_PLATFORM=amd \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_HIP_COMPILER_LAUNCHER=ccache \
            ${{ env.CMAKE_ARGS }}
          if python3 .github/scripts/cache-build-deadline.py -- cmake --build build --config Release -j "$(nproc)"; then
            :
          else
            result=$?
            if [ "$result" -eq 75 ]; then exit 0; fi
            exit "$result"
          fi
          test -s build/tools/ui/dist/index.html

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-rocm-${{ env.ROCM_VERSION }}-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Smoke package binaries
        run: |
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          rocm_short="$(echo "${ROCM_VERSION}" | cut -d '.' -f 1,2)"
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-rocm-${rocm_short}-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-rocm-7.2-x64.tar.gz
          name: beellama-bin-ubuntu-rocm-7.2-x64.tar.gz

  ubuntu-sycl:
    name: Ubuntu x64 SYCL
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-24.04
    env:
      ONEAPI_ROOT: /opt/intel/oneapi
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
      ONEAPI_INSTALLER_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh
      LEVEL_ZERO_VERSION: "1.28.2"
      LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Free up disk space
        uses: ggml-org/free-disk-space@v1.3.1
        with:
          tool-cache: true

      - name: Prepare oneAPI cache path
        run: |
          sudo mkdir -p "${ONEAPI_ROOT}"
          sudo chown -R "${USER}:${USER}" /opt/intel

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-ubuntu-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64

      - name: Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential git cmake ninja-build wget ca-certificates libssl-dev

      - name: Install oneAPI
        run: |
          installer="${RUNNER_TEMP}/intel-deep-learning-essentials-${ONEAPI_INSTALLER_VERSION}.sh"
          wget "${ONEAPI_INSTALLER_URL}" -O "${installer}"
          sudo bash "${installer}" -s -a --silent --eula accept
          sudo chown -R "${USER}:${USER}" "${ONEAPI_ROOT}"

      - name: Install Level Zero SDK
        run: |
          cd /tmp
          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
          sudo apt-get -o Dpkg::Options::="--force-overwrite" install -y ./level-zero.deb ./level-zero-devel.deb

      - name: Build
        id: build
        run: |
          source "${ONEAPI_ROOT}/setvars.sh"
          cmake -B build -G "Ninja" \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_SYCL=ON \
            -DGGML_SYCL_F16=OFF \
            -DCMAKE_C_COMPILER=icx \
            -DCMAKE_CXX_COMPILER=icpx \
            -DGGML_BACKEND_DL=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            ${{ env.CMAKE_ARGS }}
          if python3 .github/scripts/cache-build-deadline.py -- cmake --build build --config Release -j "$(nproc)"; then
            :
          else
            result=$?
            if [ "$result" -eq 75 ]; then exit 0; fi
            exit "$result"
          fi
          test -s build/tools/ui/dist/index.html

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Smoke package binaries
        run: |
          source "${ONEAPI_ROOT}/setvars.sh"
          test -s ./build/bin/libggml-sycl.so
          ldd ./build/bin/libggml-sycl.so
          mv ./build/bin/libggml-sycl.so ./build/bin/libggml-sycl.so.smoke-disabled
          trap 'mv ./build/bin/libggml-sycl.so.smoke-disabled ./build/bin/libggml-sycl.so' EXIT
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-sycl-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-sycl-x64.tar.gz
          name: beellama-bin-ubuntu-sycl-x64.tar.gz

  ubuntu-cuda:
    name: Ubuntu x64 CUDA ${{ matrix.cuda }}
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: ubuntu-24.04
    container: nvidia/cuda:${{ matrix.cuda_container }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - cuda: "12.4"
            cuda_container: "12.4.1-devel-ubuntu22.04"
            cuda_cmake_args: "-DGGML_CUDA_CUB_3DOT2=ON"
          - cuda: "13.3"
            cuda_container: "13.3.0-devel-ubuntu24.04"
            cuda_cmake_args: "-DGGML_CUDA_CUB_3DOT2=OFF"
    steps:
      - name: Dependencies
        run: |
          apt-get update
          apt-get install -y --no-install-recommends build-essential ca-certificates cmake ninja-build ccache git file libssl-dev libgomp1 python3

      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-cuda-${{ matrix.cuda }}-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-ubuntu-cuda-${{ matrix.cuda }}-x64
          sloppiness: time_macros

      - name: Build
        id: build
        run: |
          cmake -S . -B build -G Ninja \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
            -DGGML_NATIVE=OFF \
            -DGGML_CUDA=ON \
            -DGGML_CUDA_KVARN=ON \
            -DGGML_CUDA_FA=ON \
            -DGGML_BACKEND_DL=ON \
            -DGGML_CPU_ALL_VARIANTS=ON \
            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
            ${{ matrix.cuda_cmake_args }} \
            ${{ env.CMAKE_ARGS }} \
            -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined
          if python3 .github/scripts/cache-build-deadline.py -- cmake --build build --config Release -j "$(nproc)"; then
            :
          else
            result=$?
            if [ "$result" -eq 75 ]; then exit 0; fi
            exit "$result"
          fi
          test -s build/tools/ui/dist/index.html

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-ubuntu-cuda-${{ matrix.cuda }}-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Smoke package binaries
        run: |
          file ./build/bin/llama-server
          ./build/bin/llama-server --version
          ./build/bin/llama-cli --version

      - name: Pack artifacts
        run: |
          cp LICENSE ./build/bin/
          tar -czvf "beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.cuda }}-x64.tar.gz" \
            --transform "s,^\.,beellama-${{ needs.release-meta.outputs.tag_name }}," \
            -C ./build/bin .

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-${{ needs.release-meta.outputs.tag_name }}-bin-ubuntu-cuda-${{ matrix.cuda }}-x64.tar.gz
          name: beellama-bin-ubuntu-cuda-${{ matrix.cuda }}-x64.tar.gz

  windows-cpu:
    name: Windows x64 CPU
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2025
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-cpu-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-windows-cpu-x64

      - name: Install Ninja
        run: choco install ninja -y

      - name: Build
        id: build
        shell: cmd
        run: |
          for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set "VS_INSTALL_DIR=%%i"
          if not defined VS_INSTALL_DIR (
            echo No Visual Studio installation with the x64 C++ toolset was found
            exit /b 1
          )
          call "%VS_INSTALL_DIR%\VC\Auxiliary\Build\vcvarsall.bat" x64 || exit /b 1
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DGGML_NATIVE=OFF ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_CPU_ALL_VARIANTS=ON ^
            -DGGML_OPENMP=ON ^
            -DGGML_OPENMP_FETCH=ON ^
            -DCMAKE_C_COMPILER_LAUNCHER=ccache ^
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^
            %CMAKE_ARGS%
          python .github\scripts\cache-build-deadline.py -- cmake --build build --config Release -j %NUMBER_OF_PROCESSORS%
          if %errorlevel%==75 exit /b 0
          if errorlevel 1 exit /b %errorlevel%
          if not exist build\tools\ui\dist\index.html (
            echo Source-built UI output is missing
            exit /b 1
          )

      - name: Verify OpenMP is enabled
        if: ${{ steps.build.outputs.timed_out != 'true' }}
        shell: cmd
        run: |
          findstr /c:"GGML_OPENMP_ENABLED:INTERNAL=ON" build\CMakeCache.txt >nul || (
            echo GGML_OPENMP=ON was requested but OpenMP was not found; the CPU backend would ship without it
            exit /b 1
          )

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-cpu-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Pack artifacts
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $bin = ".\build\bin\Release"
          Copy-Item ".\LICENSE" $bin
          python scripts\verify-windows-package.py `
            --require-name "libomp.dll" `
            --require-name "LICENSE-LLVM-OpenMP" `
            --forbid-name "libomp140.x86_64.dll" `
            $bin

          $serverVersion = (& "$bin\llama-server.exe" --version 2>&1 | Out-String)
          if ($LASTEXITCODE -ne 0) {
            throw "llama-server --version failed with exit code $LASTEXITCODE"
          }
          Write-Host $serverVersion
          $expectedVersion = "${{ needs.release-meta.outputs.version_name }}".TrimStart("v")
          $sourceCommit = "${{ needs.release-meta.outputs.source_sha }}"
          if ($serverVersion -notmatch "version:\s+$([regex]::Escape($expectedVersion))\b") {
            throw "llama-server version does not report $expectedVersion"
          }
          $commitMatch = [regex]::Match($serverVersion, "commit\s+([0-9a-f]+)\b")
          if (-not $commitMatch.Success -or -not $sourceCommit.StartsWith($commitMatch.Groups[1].Value)) {
            throw "llama-server version does not report source commit $sourceCommit"
          }
          & "$bin\llama-cli.exe" --version
          if ($LASTEXITCODE -ne 0) {
            throw "llama-cli --version failed with exit code $LASTEXITCODE"
          }
          7z a -snl "beellama-bin-win-cpu-x64.zip" "$bin\*"

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-cpu-x64.zip
          name: beellama-bin-win-cpu-x64.zip

  windows-sycl:
    name: Windows x64 SYCL
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2022
    env:
      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
      ONEAPI_ROOT: 'C:\Program Files (x86)\Intel\oneAPI'
      ONEAPI_INSTALLER_VERSION: "2025.3.3"
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Install oneAPI
        shell: cmd
        run: scripts\install-oneapi.bat "%WINDOWS_BASEKIT_URL%" "%WINDOWS_DPCPP_MKL%"

      - name: Install Level Zero SDK
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $archive = Join-Path $env:RUNNER_TEMP "level-zero-win-sdk.zip"
          $extractDir = Join-Path $env:RUNNER_TEMP "level-zero-sdk"
          Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile $archive
          Expand-Archive -Path $archive -DestinationPath $extractDir -Force
          $sdkRoot = Get-ChildItem -Path $extractDir -Directory | Select-Object -First 1
          if (-not $sdkRoot) {
            $sdkRoot = Get-Item $extractDir
          }
          "LEVEL_ZERO_V1_SDK_PATH=$($sdkRoot.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-windows-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64

      - name: Install Ninja
        run: choco install ninja -y

      - name: Build
        id: build
        shell: cmd
        run: |
          call "%ONEAPI_ROOT%\setvars.bat" intel64 --force
          icx --version
          cmake -G "Ninja" -B build ^
            -DCMAKE_C_COMPILER=cl ^
            -DCMAKE_CXX_COMPILER=icx ^
            -DCMAKE_BUILD_TYPE=Release ^
            -DGGML_BACKEND_DL=ON ^
            -DBUILD_SHARED_LIBS=ON ^
            -DGGML_CPU=OFF ^
            -DGGML_SYCL=ON ^
            -DGGML_SYCL_F16=OFF ^
            -DLLAMA_BUILD_EXAMPLES=OFF ^
            -DLLAMA_BUILD_TESTS=OFF ^
            -DLLAMA_BUILD_TOOLS=OFF ^
            -DLLAMA_BUILD_SERVER=OFF ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DCMAKE_C_COMPILER_LAUNCHER=ccache ^
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
          python .github\scripts\cache-build-deadline.py -- cmake --build build --config Release -j %NUMBER_OF_PROCESSORS% --target ggml-sycl
          if %errorlevel%==75 exit /b 0
          if errorlevel 1 exit /b %errorlevel%

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Pack SYCL backend
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $bin = ".\build\bin"
          Copy-Item ".\LICENSE" $bin -Force

          $runtimeFiles = @(
            "$env:ONEAPI_ROOT\mkl\latest\bin\mkl_sycl_blas.5.dll",
            "$env:ONEAPI_ROOT\mkl\latest\bin\mkl_core.2.dll",
            "$env:ONEAPI_ROOT\mkl\latest\bin\mkl_tbb_thread.2.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_adapter_level_zero.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_adapter_level_zero_v2.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_adapter_opencl.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_loader.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\ur_win_proxy_loader.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\sycl8.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\svml_dispmd.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libmmd.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libiomp5md.dll",
            "$env:ONEAPI_ROOT\compiler\latest\bin\sycl-ls.exe",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libsycl-fallback-bfloat16.spv",
            "$env:ONEAPI_ROOT\compiler\latest\bin\libsycl-native-bfloat16.spv",
            "$env:ONEAPI_ROOT\dnnl\latest\bin\dnnl.dll",
            "$env:ONEAPI_ROOT\tbb\latest\bin\tbb12.dll",
            "$env:ONEAPI_ROOT\tcm\latest\bin\tcm.dll",
            "$env:ONEAPI_ROOT\tcm\latest\bin\libhwloc-15.dll",
            "$env:ONEAPI_ROOT\umf\latest\bin\umf.dll"
          )

          foreach ($file in $runtimeFiles) {
            Copy-Item -LiteralPath $file -Destination $bin -Force
          }

          $zeLoader = Get-ChildItem -Path @($env:ONEAPI_ROOT, $env:LEVEL_ZERO_V1_SDK_PATH) -Recurse -Filter "ze_loader.dll" -ErrorAction SilentlyContinue | Select-Object -First 1
          if ($zeLoader) {
            Copy-Item -LiteralPath $zeLoader.FullName -Destination $bin -Force
          } else {
            Write-Warning "ze_loader.dll was not found in oneAPI or the Level Zero SDK; relying on the system driver runtime"
          }

          python scripts\verify-windows-package.py $bin
          7z a -snl "beellama-bin-win-sycl-x64.zip" "$bin\*"

      - name: Upload SYCL backend
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-sycl-x64.zip
          name: beellama-bin-win-sycl-x64.zip

  windows-vulkan:
    name: Windows x64 Vulkan
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2025
    env:
      VULKAN_VERSION: 1.4.313.2
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Install Vulkan SDK
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $installer = Join-Path $env:RUNNER_TEMP "VulkanSDK-Installer.exe"
          curl.exe -o $installer -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
          & $installer --accept-licenses --default-answer --confirm-command install
          "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-vulkan-${{ env.VULKAN_VERSION }}-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-windows-vulkan-${{ env.VULKAN_VERSION }}-x64

      - name: Install Ninja
        run: choco install ninja -y

      - name: Build
        id: build
        shell: cmd
        run: |
          for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set "VS_INSTALL_DIR=%%i"
          if not defined VS_INSTALL_DIR (
            echo No Visual Studio installation with the x64 C++ toolset was found
            exit /b 1
          )
          call "%VS_INSTALL_DIR%\VC\Auxiliary\Build\vcvarsall.bat" x64 || exit /b 1
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_NATIVE=OFF ^
            -DGGML_CPU=OFF ^
            -DGGML_VULKAN=ON ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DCMAKE_C_COMPILER_LAUNCHER=ccache ^
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
          python .github\scripts\cache-build-deadline.py -- cmake --build build --config Release -j %NUMBER_OF_PROCESSORS% --target ggml-vulkan
          if %errorlevel%==75 exit /b 0
          if errorlevel 1 exit /b %errorlevel%

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-vulkan-${{ env.VULKAN_VERSION }}-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Pack Vulkan backend
        run: |
          7z a -snl "beellama-bin-win-vulkan-x64.zip" .\build\bin\Release\ggml-vulkan.dll

      - name: Upload Vulkan backend
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-vulkan-x64.zip
          name: beellama-bin-win-vulkan-x64.zip

  windows-cuda:
    name: Windows x64 CUDA ${{ matrix.cuda }}
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2022
    strategy:
      fail-fast: false
      matrix:
        include:
          - cuda: "12.4"
            cuda_cmake_args: "-DGGML_CUDA_CUB_3DOT2=ON"
          - cuda: "13.3"
            cuda_cmake_args: "-DGGML_CUDA_CUB_3DOT2=OFF"
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-cuda-${{ matrix.cuda }}
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-windows-cuda-${{ matrix.cuda }}
          sloppiness: time_macros

      - name: Install CUDA Toolkit
        uses: ./.github/actions/windows-setup-cuda
        with:
          cuda_version: ${{ matrix.cuda }}

      - name: Configure CUDA environment
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda }}"
          $nvcc = Join-Path $cudaPath "bin\nvcc.exe"
          if (-not (Test-Path $nvcc)) {
            throw "CUDA Toolkit not found at $cudaPath"
          }

          Join-Path $cudaPath "bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          $libNvvp = Join-Path $cudaPath "libnvvp"
          if (Test-Path $libNvvp) {
            $libNvvp | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          }

          "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          $versionEnv = "CUDA_PATH_V${{ matrix.cuda }}".Replace(".", "_")
          "$versionEnv=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

      - name: Install Ninja
        run: choco install ninja -y

      - name: Build
        id: build
        shell: cmd
        run: |
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          nvcc --version || exit /b 1
          cmake -S . -B build -G "Ninja Multi-Config" ^
            -DGGML_BACKEND_DL=ON ^
            -DGGML_NATIVE=OFF ^
            -DGGML_CPU=OFF ^
            -DGGML_CUDA=ON ^
            -DGGML_CUDA_KVARN=ON ^
            -DGGML_CUDA_FA=ON ^
            -DLLAMA_BUILD_BORINGSSL=ON ^
            -DCMAKE_C_COMPILER_LAUNCHER=ccache ^
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^
            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache ^
            ${{ matrix.cuda_cmake_args }}
          python .github\scripts\cache-build-deadline.py -- cmake --build build --config Release -j %NUMBER_OF_PROCESSORS% --target ggml-cuda
          if %errorlevel%==75 exit /b 0
          if errorlevel 1 exit /b %errorlevel%

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-cuda-${{ matrix.cuda }}
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Pack CUDA backend
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $backend = ".\build\bin\Release\ggml-cuda.dll"
          if (-not (Test-Path $backend)) {
            throw "CUDA backend was not produced at $backend"
          }
          7z a -snl "beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip" $backend

      - name: Upload CUDA backend
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
          name: beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip

      - name: Pack CUDA runtime
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $dst = ".\build\bin\cudart"
          New-Item -ItemType Directory -Force -Path $dst | Out-Null
          $roots = @("${{ env.CUDA_PATH }}\bin", "${{ env.CUDA_PATH }}\lib", "${{ env.CUDA_PATH }}\bin\x64")
          foreach ($root in $roots) {
            if (Test-Path $root) {
              Get-ChildItem -Path (Join-Path $root "*") -File -ErrorAction SilentlyContinue |
                Where-Object { $_.Name -like "cudart64_*.dll" -or $_.Name -like "cublas64_*.dll" -or $_.Name -like "cublasLt64_*.dll" } |
                Copy-Item -Destination $dst -Force
            }
          }
          foreach ($pattern in @("cudart64_*.dll", "cublas64_*.dll", "cublasLt64_*.dll")) {
            if (-not (Get-ChildItem -Path $dst -Filter $pattern -File -ErrorAction SilentlyContinue)) {
              throw "CUDA runtime package is missing $pattern"
            }
          }
          7z a "cudart-beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip" "$dst\*"

      - name: Upload CUDA runtime
        uses: actions/upload-artifact@v6
        with:
          path: cudart-beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
          name: cudart-beellama-bin-win-cuda-${{ matrix.cuda }}-x64.zip

  windows-hip:
    name: Windows x64 HIP/Radeon
    needs: release-meta
    if: ${{ needs.release-meta.outputs.build_packages == 'true' }}
    runs-on: windows-2022
    env:
      HIPSDK_INSTALLER_VERSION: "26.Q1"
      GPU_TARGETS: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
    steps:
      - name: Reserve cache save time
        shell: bash
        run: echo "CACHE_SAVE_DEADLINE=$(($(date +%s) + 330 * 60))" >> "$GITHUB_ENV"

      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Grab rocWMMA package
        run: |
          curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb"
          7z x rocwmma.deb
          7z x data.tar

      - name: ccache
        uses: ./.github/actions/setup-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-x64
          restore-keys: release-${{ needs.release-meta.outputs.cache_parent_channel }}-windows-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-x64

      - name: Install ROCm
        run: |
          $ErrorActionPreference = "Stop"
          Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-Win11-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe"
          $installTimeoutSeconds = 1800
          $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -WindowStyle Hidden -PassThru
          $completed = $proc.WaitForExit($installTimeoutSeconds * 1000)
          if (-not $completed) {
            $proc.Kill()
            Write-Error "ROCm installation timed out after $($installTimeoutSeconds / 60) minutes"
          }
          if ($proc.ExitCode -ne 0) {
            Write-Error "ROCm installation failed with exit code $($proc.ExitCode)"
          }

      - name: Verify ROCm
        run: |
          $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1
          if (-not $clangPath) {
            Write-Error "ROCm installation not found"
          }
          & $clangPath.FullName --version

      - name: Build
        id: build
        run: |
          $ErrorActionPreference = "Stop"
          $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Split-Path | Split-Path)
          $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
          cmake -G "Unix Makefiles" -B build -S . `
            -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" `
            -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `
            -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.2.1/include/ -Wno-ignored-attributes -Wno-nested-anon-types" `
            -DCMAKE_BUILD_TYPE=Release `
            -DGGML_BACKEND_DL=ON `
            -DGGML_NATIVE=OFF `
            -DGGML_CPU=OFF `
            -DGPU_TARGETS="${env:GPU_TARGETS}" `
            -DGGML_HIP=ON `
            -DGGML_CUDA_KVARN=ON `
            -DGGML_CUDA_FA=ON `
            -DCMAKE_C_COMPILER_LAUNCHER=ccache `
            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
            -DCMAKE_HIP_COMPILER_LAUNCHER=ccache `
            -DLLAMA_BUILD_BORINGSSL=ON
          $buildJobs = [int]$env:NUMBER_OF_PROCESSORS
          python .github/scripts/cache-build-deadline.py -- cmake --build build --target ggml-hip -j $buildJobs
          if ($LASTEXITCODE -eq 75) { exit 0 }
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          New-Item -ItemType Directory -Force -Path "build\bin\rocblas\library" | Out-Null
          New-Item -ItemType Directory -Force -Path "build\bin\hipblaslt\library" | Out-Null
          Copy-Item "${env:HIP_PATH}\bin\libhipblas.dll" "build\bin\"
          Copy-Item "${env:HIP_PATH}\bin\libhipblaslt.dll" "build\bin\"
          Copy-Item "${env:HIP_PATH}\bin\rocblas.dll" "build\bin\"
          Copy-Item "${env:HIP_PATH}\bin\amdhip64_7.dll" "build\bin\"
          Copy-Item "${env:HIP_PATH}\bin\amd_comgr_3.dll" "build\bin\"
          $kpack = Join-Path $env:HIP_PATH "bin\rocm_kpack.dll"
          if (Test-Path $kpack) {
            Copy-Item $kpack "build\bin\"
          } else {
            Write-Host "ROCm SDK does not supply rocm_kpack.dll; verifying that packaged binaries do not import it"
          }
          Copy-Item "${env:HIP_PATH}\bin\rocblas\library\*" "build\bin\rocblas\library\"
          Copy-Item "${env:HIP_PATH}\bin\hipblaslt\library\*" "build\bin\hipblaslt\library\"

      - name: ccache stats
        if: always()
        run: ccache --show-stats

      - name: Save ccache
        # Only save once the build process tree has exited. A controlled build
        # deadline keeps partial ccache progress without archiving live writes.
        if: ${{ success() && !cancelled() }}
        uses: ./.github/actions/save-rolling-ccache
        with:
          key: release-${{ needs.release-meta.outputs.cache_channel }}-windows-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-x64
          ref: refs/heads/main
          evict-old-files: 1d

      - name: Report incomplete build
        if: ${{ !cancelled() && steps.build.outputs.timed_out == 'true' }}
        run: exit 1

      - name: Pack artifacts
        run: |
          python scripts\verify-windows-package.py `
            --require-name "ggml-hip.dll" `
            --require-name "amdhip64_7.dll" `
            --require-name "amd_comgr_3.dll" `
            --require-local-import "rocm_kpack.dll" `
            .\build\bin
          7z a -snl "beellama-bin-win-hip-radeon-x64.zip" .\build\bin\*

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          path: beellama-bin-win-hip-radeon-x64.zip
          name: beellama-bin-win-hip-radeon-x64.zip

  package-assets:
    name: Assemble release assets
    needs:
      - release-meta
      - macos-arm64
      - ubuntu-cpu
      - ubuntu-vulkan
      - ubuntu-rocm
      - ubuntu-sycl
      - ubuntu-cuda
      - windows-cpu
      - windows-sycl
      - windows-vulkan
      - windows-cuda
      - windows-hip
    runs-on: ubuntu-24.04
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Download package artifacts
        uses: actions/download-artifact@v7
        with:
          pattern: beellama-bin-*
          path: ./artifact
          merge-multiple: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
          run-id: ${{ github.run_id }}
          repository: ${{ github.repository }}

      - name: Download CUDA runtime artifacts
        uses: actions/download-artifact@v7
        with:
          pattern: cudart-beellama-*
          path: ./artifact
          merge-multiple: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
          run-id: ${{ github.run_id }}
          repository: ${{ github.repository }}

      - name: Prepare release assets
        shell: bash
        env:
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
        run: |
          set -euo pipefail
          shopt -s nullglob

          mkdir -p release

          cpu_zip="artifact/beellama-bin-win-cpu-x64.zip"
          if [[ -f "${cpu_zip}" ]]; then
            temp_dir="$(mktemp -d)"
            unzip -q "${cpu_zip}" -d "${temp_dir}"
            for target_zip in artifact/beellama-bin-win-*-x64.zip; do
              if [[ "$(basename "${target_zip}")" == "beellama-bin-win-cpu-x64.zip" ]]; then
                continue
              fi
              realpath_target_zip="$(realpath "${target_zip}")"
              (cd "${temp_dir}" && zip -qr "${realpath_target_zip}" .)
            done
            rm -rf "${temp_dir}"
          fi

          for zip_file in artifact/beellama-bin-win-*.zip; do
            base_name="$(basename "${zip_file}" .zip)"
            mv "${zip_file}" "release/beellama-${TAG_NAME}-${base_name#beellama-}.zip"
          done

          for zip_file in artifact/cudart-beellama-bin-win-cuda-*.zip; do
            base_name="$(basename "${zip_file}" .zip)"
            suffix="${base_name#cudart-beellama-bin-win-}"
            mv "${zip_file}" "release/beellama-${TAG_NAME}-cudart-win-${suffix}.zip"
          done

          for tar_file in artifact/*.tar.gz; do
            mv "${tar_file}" release/
          done

          python3 scripts/verify-windows-package.py \
            --require-name "llama-server.exe" \
            --require-name "llama-cli.exe" \
            --require-name "libomp.dll" \
            --forbid-name "libomp140.x86_64.dll" \
            release/beellama-${TAG_NAME}-bin-win-*.zip
          for cuda_version in 12.4 13.3; do
            package_zip="release/beellama-${TAG_NAME}-bin-win-cuda-${cuda_version}-x64.zip"
            runtime_zip="release/beellama-${TAG_NAME}-cudart-win-cuda-${cuda_version}-x64.zip"
            if [[ -f "${package_zip}" && -f "${runtime_zip}" ]]; then
              temp_dir="$(mktemp -d)"
              unzip -q "${package_zip}" -d "${temp_dir}"
              unzip -q "${runtime_zip}" -d "${temp_dir}"
              python3 scripts/verify-windows-package.py \
                --require-name "ggml-cuda.dll" \
                --require-glob "cudart64_*.dll" \
                --require-glob "cublas64_*.dll" \
                --require-glob "cublasLt64_*.dll" \
                "${temp_dir}"
              rm -rf "${temp_dir}"
            fi
          done
          sha256sum release/* > release/SHA256SUMS.txt
          ls -lh release

      - name: Write release notes
        shell: bash
        env:
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
          VERSION_NAME: ${{ needs.release-meta.outputs.version_name }}
          PREVIEW: ${{ needs.release-meta.outputs.preview }}
          SOURCE_REF: ${{ needs.release-meta.outputs.source_ref }}
          SOURCE_SHA: ${{ needs.release-meta.outputs.source_sha }}
          SHORT_SHA: ${{ needs.release-meta.outputs.short_sha }}
          IMAGE_REPO: ${{ needs.release-meta.outputs.image_repo }}
        run: |
          set -euo pipefail
          release_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/download/${TAG_NAME}"
          changelog_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_SHA}/CHANGELOG.md"
          repo_name="${GITHUB_REPOSITORY#*/}"
          repo_name="${repo_name,,}"
          package_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pkgs/container/${repo_name}"
          python3 scripts/extract-changelog.py \
            --tag "${VERSION_NAME}" \
            --changelog CHANGELOG.md \
            --fallback-url "${changelog_url}" \
            --output changelog-fragment.md
          python3 scripts/write-release-notes.py \
            --tag "${TAG_NAME}" \
            --release-url "${release_url}" \
            --image-repo "${IMAGE_REPO}" \
            --package-url "${package_url}" \
            --changelog changelog-fragment.md \
            --output release-notes.md
          if [[ "${PREVIEW}" == "true" ]]; then
            {
              echo "> [!WARNING]"
              echo "> Rolling preview build from [\`${SOURCE_REF}\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_REF}) at [\`${SHORT_SHA}\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${SOURCE_SHA}). Assets and moving Docker tags are replaced by newer successful branch builds. Last updated: $(date -u +'%d/%m/%Y %H:%M')."
              echo
              cat release-notes.md
            } > release-notes-preview.md
            mv release-notes-preview.md release-notes.md
          fi

      - name: Upload release assets
        uses: actions/upload-artifact@v6
        with:
          name: beellama-release-assets
          path: |
            release/*
            release-notes.md

  docker-build:
    name: Docker (${{ matrix.config.display }})
    if: ${{ needs.release-meta.outputs.publish_docker == 'true' }}
    needs:
      - release-meta
      - ubuntu-cpu
      - ubuntu-vulkan
      - ubuntu-rocm
      - ubuntu-sycl
      - ubuntu-cuda
    runs-on: ${{ matrix.config.runs_on }}
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: cpu-amd64
            display: CPU amd64
            group: cpu
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: ubuntu:24.04
            runtime_packages: ca-certificates curl ffmpeg libgomp1
            runs_on: ubuntu-24.04
          - name: cpu-arm64
            display: CPU arm64
            group: cpu
            arch: arm64
            platform: linux/arm64
            artifact: beellama-bin-ubuntu-arm64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: ubuntu:24.04
            runtime_packages: ca-certificates curl ffmpeg libgomp1
            runs_on: ubuntu-24.04-arm
          - name: cuda12
            display: CUDA 12.4
            group: cuda12
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-cuda-12.4-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: nvidia/cuda:12.4.1-runtime-ubuntu22.04
            runtime_packages: ca-certificates curl ffmpeg libgomp1
            runs_on: ubuntu-24.04
          - name: cuda13
            display: CUDA 13.3
            group: cuda13
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-cuda-13.3-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: nvidia/cuda:13.3.0-runtime-ubuntu24.04
            runtime_packages: ca-certificates curl ffmpeg libgomp1
            runs_on: ubuntu-24.04
          - name: rocm
            display: ROCm
            group: rocm
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-rocm-7.2-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: rocm/dev-ubuntu-24.04:7.2.1-complete
            runtime_packages: ca-certificates curl ffmpeg libgomp1
            runs_on: ubuntu-24.04
          - name: vulkan
            display: Vulkan
            group: vulkan
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-vulkan-x64.tar.gz
            dockerfile: .devops/runtime-server.Dockerfile
            base_image: ubuntu:24.04
            runtime_packages: ca-certificates curl ffmpeg libegl1 libgl1 libgles2 libglvnd0 libglx0 libgomp1 libvulkan1 mesa-vulkan-drivers
            runs_on: ubuntu-24.04
          - name: sycl
            display: SYCL
            group: sycl
            arch: amd64
            platform: linux/amd64
            artifact: beellama-bin-ubuntu-sycl-x64.tar.gz
            dockerfile: .devops/runtime-intel-server.Dockerfile
            base_image: unused
            runtime_packages: unused
            runs_on: ubuntu-24.04
    permissions:
      contents: read
      packages: write
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Download package artifact
        uses: actions/download-artifact@v7
        with:
          name: ${{ matrix.config.artifact }}
          path: ./package

      - name: Prepare Docker context
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p docker-context/app
          tar_file="$(find package -type f -name '*.tar.gz' | head -n 1)"
          if [[ -z "${tar_file}" ]]; then
            echo "No tar.gz package found for ${{ matrix.config.artifact }}" >&2
            exit 1
          fi
          tar -xzf "${tar_file}" -C docker-context/app --strip-components=1
          test -x docker-context/app/llama-server
          cp "${{ matrix.config.dockerfile }}" docker-context/Dockerfile
          ls -lh docker-context/app/llama-server

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4

      - name: Log in to GHCR
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Get build date
        id: build_date
        run: echo "date=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> "${GITHUB_OUTPUT}"

      - name: Build and push image digest
        id: build
        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
        with:
          context: ./docker-context
          file: ./docker-context/Dockerfile
          platforms: ${{ matrix.config.platform }}
          outputs: type=image,name=${{ needs.release-meta.outputs.image_repo }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true
          provenance: false
          build-args: |
            BASE_IMAGE=${{ matrix.config.base_image }}
            RUNTIME_PACKAGES=${{ matrix.config.runtime_packages }}
            BUILD_DATE=${{ steps.build_date.outputs.date }}
            APP_VERSION=${{ needs.release-meta.outputs.tag_name }}
            APP_REVISION=${{ needs.release-meta.outputs.source_sha }}
            IMAGE_URL=${{ github.server_url }}/${{ github.repository }}
            IMAGE_SOURCE=${{ github.server_url }}/${{ github.repository }}
          annotations: |
            manifest:org.opencontainers.image.created=${{ steps.build_date.outputs.date }}
            manifest:org.opencontainers.image.version=${{ needs.release-meta.outputs.tag_name }}
            manifest:org.opencontainers.image.revision=${{ needs.release-meta.outputs.source_sha }}
            manifest:org.opencontainers.image.title=BeeLlama.cpp
            manifest:org.opencontainers.image.description=BeeLlama.cpp GGUF inference with upstream DFlash and KVarN KV-cache support
            manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}
            manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
          cache-from: ${{ matrix.config.name != 'rocm' && format('type=registry,ref={0}:buildcache-runtime-{1}', needs.release-meta.outputs.image_repo, matrix.config.name) || '' }}
          cache-to: ${{ matrix.config.name != 'rocm' && format('type=registry,ref={0}:buildcache-runtime-{1},mode=max', needs.release-meta.outputs.image_repo, matrix.config.name) || '' }}

      - name: Record digest
        shell: bash
        run: |
          set -euo pipefail
          digest="${{ steps.build.outputs.digest }}"
          if [[ -z "${digest}" ]]; then
            echo "Docker build did not return a digest." >&2
            exit 1
          fi
          mkdir -p /tmp/docker-digests
          printf '%s\t%s\t%s\n' "${{ matrix.config.group }}" "${{ matrix.config.arch }}" "${digest}" > "/tmp/docker-digests/${{ matrix.config.name }}.tsv"

      - name: Upload digest metadata
        uses: actions/upload-artifact@v6
        with:
          name: docker-digest-${{ matrix.config.name }}
          path: /tmp/docker-digests/${{ matrix.config.name }}.tsv
          if-no-files-found: error

  docker-merge:
    name: Docker tags (${{ matrix.config.display }})
    if: ${{ needs.release-meta.outputs.publish_docker == 'true' }}
    needs:
      - release-meta
      - docker-build
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        config:
          - group: cpu
            display: CPU
            platforms: linux/amd64 linux/arm64
            tag_suffixes: none -cpu
          - group: cuda12
            display: CUDA 12.4
            platforms: linux/amd64
            tag_suffixes: -cuda -cuda12
          - group: cuda13
            display: CUDA 13.3
            platforms: linux/amd64
            tag_suffixes: -cuda13
          - group: rocm
            display: ROCm
            platforms: linux/amd64
            tag_suffixes: -rocm
          - group: vulkan
            display: Vulkan
            platforms: linux/amd64
            tag_suffixes: -vulkan
          - group: sycl
            display: SYCL
            platforms: linux/amd64
            tag_suffixes: -sycl
    steps:
      - name: Download digest metadata
        uses: actions/download-artifact@v7
        with:
          pattern: docker-digest-*
          path: /tmp/docker-digests
          merge-multiple: true

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4

      - name: Log in to GHCR
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Create Docker tags
        shell: bash
        env:
          IMAGE_REPO: ${{ needs.release-meta.outputs.image_repo }}
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
          STABLE_RELEASE: ${{ needs.release-meta.outputs.stable_release }}
          SOURCE_REF: ${{ needs.release-meta.outputs.source_ref }}
          SOURCE_SHA: ${{ needs.release-meta.outputs.source_sha }}
          SHORT_SHA: ${{ needs.release-meta.outputs.short_sha }}
          GROUP: ${{ matrix.config.group }}
          PLATFORMS: ${{ matrix.config.platforms }}
          TAG_SUFFIXES: ${{ matrix.config.tag_suffixes }}
        run: |
          set -euo pipefail

          if [[ "${STABLE_RELEASE}" != "true" ]]; then
            current_sha="$(git ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/heads/${SOURCE_REF}" | awk '{ print $1 }')"
            if [[ -z "${current_sha}" || "${current_sha}" != "${SOURCE_SHA}" ]]; then
              echo "Branch ${SOURCE_REF} moved from ${SOURCE_SHA} to ${current_sha:-missing}; refusing to publish stale preview Docker tags." >&2
              exit 1
            fi
          fi

          digest_glob="/tmp/docker-digests/*.tsv"
          if ! ls ${digest_glob} >/dev/null 2>&1; then
            echo "No Docker digest metadata found." >&2
            exit 1
          fi

          sources=()
          for platform in ${PLATFORMS}; do
            arch="${platform#linux/}"
            digest="$(awk -F '\t' -v g="${GROUP}" -v a="${arch}" '$1 == g && $2 == a { print $3; exit }' ${digest_glob})"
            if [[ -z "${digest}" ]]; then
              echo "Missing digest for group=${GROUP} arch=${arch}" >&2
              exit 1
            fi
            sources+=("${IMAGE_REPO}@${digest}")
          done

          tag_args=()
          for suffix in ${TAG_SUFFIXES}; do
            if [[ "${suffix}" == "none" ]]; then
              suffix=""
            fi
            if [[ "${STABLE_RELEASE}" == "true" ]]; then
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}")
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}-${TAG_NAME}")
            else
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}-${TAG_NAME}")
              tag_args+=("-t" "${IMAGE_REPO}:server${suffix}-${TAG_NAME}-${SHORT_SHA}")
            fi
          done

          printf 'Creating Docker tags:\n'
          printf '  %s\n' "${tag_args[@]}"
          docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"

  release:
    name: Publish GitHub release
    needs:
      - release-meta
      - package-assets
      - docker-merge
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    steps:
      - name: Clone
        uses: actions/checkout@v6
        with:
          ref: ${{ needs.release-meta.outputs.source_sha }}
          fetch-depth: 0

      - name: Prepare publication tag
        env:
          PREVIEW: ${{ needs.release-meta.outputs.preview }}
          SOURCE_REF: ${{ needs.release-meta.outputs.source_ref }}
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
          SOURCE_SHA: ${{ needs.release-meta.outputs.source_sha }}
        run: |
          set -euo pipefail
          if [[ "${PREVIEW}" == "true" ]]; then
            current_sha="$(git ls-remote origin "refs/heads/${SOURCE_REF}" | awk '{ print $1 }')"
            if [[ -z "${current_sha}" || "${current_sha}" != "${SOURCE_SHA}" ]]; then
              echo "Branch ${SOURCE_REF} moved from ${SOURCE_SHA} to ${current_sha:-missing}; refusing to publish stale preview assets." >&2
              exit 1
            fi
            git tag -f "${TAG_NAME}" "${SOURCE_SHA}"
            git push origin "refs/tags/${TAG_NAME}" --force
          else
            git fetch origin +refs/tags/"${TAG_NAME}":refs/tags/"${TAG_NAME}"
            tag_commit="$(git rev-list -n 1 "${TAG_NAME}")"
            if [[ "${tag_commit}" != "${SOURCE_SHA}" ]]; then
              echo "Tag ${TAG_NAME} moved from ${SOURCE_SHA} to ${tag_commit}; refusing to publish." >&2
              exit 1
            fi
          fi

      - name: Download release assets
        uses: actions/download-artifact@v7
        with:
          name: beellama-release-assets
          path: ./release-bundle

      - name: Create or update release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG_NAME: ${{ needs.release-meta.outputs.tag_name }}
          RELEASE_TITLE: ${{ needs.release-meta.outputs.release_title }}
          PREVIEW: ${{ needs.release-meta.outputs.preview }}
          SOURCE_SHA: ${{ needs.release-meta.outputs.source_sha }}
        run: |
          set -euo pipefail
          notes_file="$(find release-bundle -type f -name release-notes.md | head -n 1)"
          if [[ -z "${notes_file}" ]]; then
            echo "release-notes.md not found in release asset bundle" >&2
            exit 1
          fi

          mapfile -t release_files < <(find release-bundle -type f ! -name release-notes.md | sort)
          if [[ "${#release_files[@]}" -eq 0 ]]; then
            echo "No release files found in release asset bundle" >&2
            exit 1
          fi

          if gh release view "${TAG_NAME}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
            gh release upload "${TAG_NAME}" "${release_files[@]}" --clobber --repo "${GITHUB_REPOSITORY}"
            mapfile -t existing_assets < <(gh release view "${TAG_NAME}" --repo "${GITHUB_REPOSITORY}" --json assets --jq '.assets[].name')
            for existing_asset in "${existing_assets[@]}"; do
              keep_asset="false"
              for release_file in "${release_files[@]}"; do
                if [[ "$(basename "${release_file}")" == "${existing_asset}" ]]; then
                  keep_asset="true"
                  break
                fi
              done
              if [[ "${keep_asset}" != "true" ]]; then
                gh release delete-asset "${TAG_NAME}" "${existing_asset}" --yes --repo "${GITHUB_REPOSITORY}"
              fi
            done
            if [[ "${PREVIEW}" == "true" ]]; then
              gh release edit "${TAG_NAME}" --repo "${GITHUB_REPOSITORY}" --title "${RELEASE_TITLE}" --notes-file "${notes_file}" --target "${SOURCE_SHA}" --prerelease --latest=false --verify-tag
            else
              gh release edit "${TAG_NAME}" --repo "${GITHUB_REPOSITORY}" --title "${RELEASE_TITLE}" --notes-file "${notes_file}" --target "${SOURCE_SHA}" --prerelease=false --latest=true --verify-tag
            fi
          else
            if [[ "${PREVIEW}" == "true" ]]; then
              gh release create "${TAG_NAME}" "${release_files[@]}" --repo "${GITHUB_REPOSITORY}" --title "${RELEASE_TITLE}" --notes-file "${notes_file}" --target "${SOURCE_SHA}" --prerelease --latest=false --verify-tag
            else
              gh release create "${TAG_NAME}" "${release_files[@]}" --repo "${GITHUB_REPOSITORY}" --title "${RELEASE_TITLE}" --notes-file "${notes_file}" --target "${SOURCE_SHA}" --latest=true --verify-tag
            fi
          fi