diff --git a/.ecrc b/.ecrc
index c68877ec211f..8204ad174269 100644
--- a/.ecrc
+++ b/.ecrc
@@ -1,5 +1,5 @@
{
- "Exclude": ["^\\.gitmodules$", "stb_image\\.h"],
+ "Exclude": ["^\\.gitmodules$", "stb_image\\.h", "^vendor/"],
"Disable": {
"IndentSize": true
}
diff --git a/.flake8 b/.flake8
index 669d231f1f63..16de5486797d 100644
--- a/.flake8
+++ b/.flake8
@@ -15,4 +15,5 @@ exclude =
build,
# This contains builds that we don't want to check
dist # This is generated with `python build .` for package releases
+ scripts/tune
# max-complexity = 10
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000000..fc7b35173a93
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @tetherto/ai-runtime-bk-models
diff --git a/.github/workflows/approval-check-worker.yml b/.github/workflows/approval-check-worker.yml
new file mode 100644
index 000000000000..3a60b0bff53f
--- /dev/null
+++ b/.github/workflows/approval-check-worker.yml
@@ -0,0 +1,19 @@
+name: Approval Check Worker
+
+on:
+ pull_request_review:
+ types: [submitted, dismissed]
+
+jobs:
+ check-approvals:
+ permissions:
+ contents: write
+ pull-requests: write
+ statuses: write
+ issues: write
+
+ uses: tetherto/qvac-devops/.github/workflows/approval-check-worker.yml@production-workflows-tag
+ secrets: inherit
+ with:
+ pr_number: ${{ github.event.pull_request.number }}
+ pr_sha: ${{ github.event.pull_request.head.sha }}
diff --git a/.github/workflows/build-cmake-pkg.yml b/.github/workflows/build-cmake-pkg.yml
index fee2ab96bd0e..fa604bb6f47b 100644
--- a/.github/workflows/build-cmake-pkg.yml
+++ b/.github/workflows/build-cmake-pkg.yml
@@ -25,7 +25,7 @@ jobs:
cmake --build build --config Release
cmake --install build --prefix "$PREFIX" --config Release
- export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
+ export LLAMA_CONFIG="$PREFIX"/share/llama/llama-config.cmake
tclsh <<'EOF'
set build(commit) [string trim [exec git rev-parse --short HEAD]]
set build(number) [string trim [exec git rev-list --count HEAD]]
@@ -47,5 +47,5 @@ jobs:
EOF
cd examples/simple-cmake-pkg
- cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
+ cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/share
cmake --build build
diff --git a/.github/workflows/build-linux-cross.yml b/.github/workflows/build-linux-cross.yml
index 36201281f005..421442e313fe 100644
--- a/.github/workflows/build-linux-cross.yml
+++ b/.github/workflows/build-linux-cross.yml
@@ -4,6 +4,7 @@ on:
workflow_call:
jobs:
+ # Disabled. Fails to install some dependencies from arch-specific repositories.
# ubuntu-24-riscv64-cpu-cross:
# runs-on: ubuntu-24.04
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 49e836d9b202..9d87137a5950 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -8,7 +8,9 @@ on:
paths: [
'.github/workflows/build.yml',
'.github/workflows/build-linux-cross.yml',
- '.github/workflows/build-cmake-pkg.yml',
+ # Disable. There are some modifications in the fork
+ # so that ggml dynamic builds work with vcpkg.
+ # '.github/workflows/build-cmake-pkg.yml',
'**/CMakeLists.txt',
'**/.cmake',
'**/*.h',
@@ -28,7 +30,9 @@ on:
paths: [
'.github/workflows/build.yml',
'.github/workflows/build-linux-cross.yml',
- '.github/workflows/build-cmake-pkg.yml',
+ # Disable. There are some modifications in the fork
+ # so that ggml dynamic builds work with vcpkg.
+ # '.github/workflows/build-cmake-pkg.yml',
'**/CMakeLists.txt',
'**/.cmake',
'**/*.h',
@@ -117,8 +121,9 @@ jobs:
-DLLAMA_CURL=OFF \
-DLLAMA_BUILD_BORINGSSL=ON \
-DGGML_METAL=OFF \
- -DGGML_RPC=ON \
- -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3 \
+ -DGGML_BLAS=OFF \
+ -DGGML_RPC=ON
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
- name: Test
@@ -269,11 +274,12 @@ jobs:
id: checkout
uses: actions/checkout@v4
- - name: ccache
- uses: ggml-org/ccache-action@v1.2.16
- with:
- key: ubuntu-latest-cmake-sanitizer-${{ matrix.sanitizer }}
- evict-old-files: 1d
+ # ccache disabled for sanitizer builds to ensure clean builds with correct sanitizer flags
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.16
+ # with:
+ # key: ubuntu-latest-cmake-sanitizer-${{ matrix.sanitizer }}
+ # evict-old-files: 1d
- name: Dependencies
id: depends
@@ -308,6 +314,16 @@ jobs:
- name: Test
id: cmake_test
+ env:
+ # AddressSanitizer options
+ ASAN_OPTIONS: "verbosity=1:abort_on_error=1:print_stats=1:check_initialization_order=1:strict_init_order=1:detect_stack_use_after_return=1:print_summary=1:print_scariness=1:print_legend=1"
+ # ThreadSanitizer options
+ TSAN_OPTIONS: "verbosity=1:abort_on_error=1:print_stats=1:print_summary=1:print_legend=1"
+ # UndefinedBehaviorSanitizer options
+ # Note: abort_on_error=0 allows UBSAN to print full diagnostics before aborting
+ UBSAN_OPTIONS: "verbosity=2:abort_on_error=1:print_stacktrace=1:print_summary=1:halt_on_error=1:report_error_type=1:silence_unsigned_overflow=0"
+ # Common options for all sanitizers
+ MSAN_OPTIONS: "verbosity=1:abort_on_error=1:print_stats=1"
run: |
cd build
ctest -L main --verbose --timeout 900
@@ -354,6 +370,7 @@ jobs:
id: checkout
uses: actions/checkout@v4
+ # Reduce false pass/failure on CI
# - name: ccache
# uses: ggml-org/ccache-action@v1.2.16
# with:
@@ -389,11 +406,12 @@ jobs:
id: checkout
uses: actions/checkout@v4
- - name: ccache
- uses: ggml-org/ccache-action@v1.2.16
- with:
- key: ubuntu-24-cmake-vulkan-deb
- evict-old-files: 1d
+ # Reduce false pass/failure on CI
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.16
+ # with:
+ # key: ubuntu-24-cmake-vulkan
+ # evict-old-files: 1d
- name: Dependencies
id: depends
@@ -417,7 +435,7 @@ jobs:
cmake --build build -j $(nproc)
ubuntu-24-cmake-vulkan:
- runs-on: ubuntu-24.04
+ runs-on: ai-run-linux-gpu
steps:
- name: Clone
@@ -435,7 +453,7 @@ jobs:
run: |
sudo add-apt-repository -y ppa:kisak/kisak-mesa
sudo apt-get update -y
- sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
+ sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev cmake
- name: Get latest Vulkan SDK version
id: vulkan_sdk_version
@@ -483,11 +501,12 @@ jobs:
id: checkout
uses: actions/checkout@v4
- - name: ccache
- uses: ggml-org/ccache-action@v1.2.16
- with:
- key: ubuntu-24-cmake-webgpu
- evict-old-files: 1d
+ # Reduce false pass/failure on CI
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.16
+ # with:
+ # key: ubuntu-24-cmake-webgpu
+ # evict-old-files: 1d
- name: Dependencies
id: depends
@@ -602,11 +621,12 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev
- - name: ccache
- uses: ggml-org/ccache-action@v1.2.16
- with:
- key: ubuntu-22-cmake-hip
- evict-old-files: 1d
+ # Reduce false pass/failure on CI
+ # - name: ccache
+ # uses: ggml-org/ccache-action@v1.2.16
+ # with:
+ # key: ubuntu-22-cmake-hip
+ # evict-old-files: 1d
- name: Build with native CMake HIP support
id: cmake_build
@@ -753,8 +773,10 @@ jobs:
build-linux-cross:
uses: ./.github/workflows/build-linux-cross.yml
- build-cmake-pkg:
- uses: ./.github/workflows/build-cmake-pkg.yml
+ # Disable. There are some modifications in the fork
+ # so that ggml dynamic builds work with vcpkg.
+ # build-cmake-pkg:
+ # uses: ./.github/workflows/build-cmake-pkg.yml
macOS-latest-cmake-ios:
runs-on: macos-latest
@@ -1325,6 +1347,8 @@ jobs:
include:
- build: 'arm64-cpu'
defines: '-D ANDROID_ABI=arm64-v8a -D ANDROID_PLATFORM=android-31 -D CMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -D GGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm -G Ninja -D LLAMA_CURL=OFF -D GGML_OPENMP=OFF'
+ - build: 'arm64-vulkan'
+ defines: '-DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-31 -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm -DGGML_VULKAN=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_CURL=OFF -DGGML_OPENMP=OFF -DVulkan_INCLUDE_DIR=${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include -DVulkan_LIBRARY=${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/31/libvulkan.so -G Ninja'
- build: 'arm64-snapdragon'
defines: '--preset arm64-android-snapdragon-release'
@@ -1333,6 +1357,36 @@ jobs:
id: checkout
uses: actions/checkout@v4
+ - name: Get latest Vulkan SDK version
+ id: vulkan_sdk_version
+ if: ${{ matrix.build == 'arm64-vulkan' }}
+ run: |
+ echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
+
+ - name: Use Vulkan SDK Cache
+ uses: actions/cache@v4
+ id: cache-sdk
+ if: ${{ matrix.build == 'arm64-vulkan' }}
+ with:
+ path: ./vulkan_sdk
+ key: vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
+
+ - name: Setup Vulkan SDK
+ if: ${{ matrix.build == 'arm64-vulkan' && steps.cache-sdk.outputs.cache-hit != 'true' }}
+ uses: ./.github/actions/linux-setup-vulkan
+ with:
+ path: ./vulkan_sdk
+ version: ${{ env.VULKAN_SDK_VERSION }}
+
+ - name: Configure Android Vulkan shader toolchain
+ id: android_vulkan_toolchain
+ if: ${{ matrix.build == 'arm64-vulkan' }}
+ run: |
+ test -x "$PWD/vulkan_sdk/bin/glslc"
+ test -r "${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/31/libvulkan.so"
+ echo "QVAC_VULKAN_GLSLC=$PWD/vulkan_sdk/bin/glslc" >> "$GITHUB_ENV"
+ "$PWD/vulkan_sdk/bin/glslc" --version
+
- name: Install OpenCL Headers and Libs
id: install_opencl
if: ${{ matrix.build == 'arm64-snapdragon' }}
@@ -1380,10 +1434,21 @@ jobs:
- name: Build
id: ndk_build
run: |
- cmake ${{ matrix.defines }} -B build
+ cmake ${{ matrix.defines }} ${QVAC_VULKAN_GLSLC:+-DVulkan_GLSLC_EXECUTABLE=$QVAC_VULKAN_GLSLC} -B build
cmake --build build
cmake --install build --prefix pkg-adb/llama.cpp
+ - name: Pack Android NDK artifacts
+ id: pack_ndk_artifacts
+ run: |
+ tar -czf llama-android-${{ matrix.build }}.tar.gz -C pkg-adb llama.cpp
+
+ - name: Upload Android NDK artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-android-${{ matrix.build }}.tar.gz
+ name: llama-android-${{ matrix.build }}.tar.gz
+
- name: Test
id: cmake_test
run: |
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 03cd41c57026..a0e03b5bf936 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -18,7 +18,7 @@ concurrency:
env:
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
- CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"
+ CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON"
jobs:
macOS-arm64:
@@ -148,8 +148,6 @@ jobs:
include:
- build: 'x64'
os: ubuntu-22.04
- - build: 's390x'
- os: ubuntu-24.04-s390x
# GGML_BACKEND_DL and GGML_CPU_ALL_VARIANTS are not currently supported on arm
# - build: 'arm64'
# os: ubuntu-22.04-arm
@@ -348,10 +346,6 @@ jobs:
arch: 'x64'
defines: '-DGGML_VULKAN=ON'
target: 'ggml-vulkan'
- - backend: 'opencl-adreno'
- arch: 'arm64'
- defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'
- target: 'ggml-opencl'
steps:
- name: Clone
@@ -379,26 +373,6 @@ jobs:
run: |
choco install ninja
- - name: Install OpenCL Headers and Libs
- id: install_opencl
- if: ${{ matrix.backend == 'opencl-adreno' && matrix.arch == 'arm64' }}
- run: |
- git clone https://github.com/KhronosGroup/OpenCL-Headers
- cd OpenCL-Headers
- cmake -B build `
- -DBUILD_TESTING=OFF `
- -DOPENCL_HEADERS_BUILD_TESTING=OFF `
- -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
- -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
- cmake --build build --target install
- git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
- cd OpenCL-ICD-Loader
- cmake -B build-arm64-release `
- -A arm64 `
- -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
- -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
- cmake --build build-arm64-release --target install --config release
-
- name: Build
id: cmake_build
run: |
@@ -408,7 +382,7 @@ jobs:
- name: Pack artifacts
id: pack_artifacts
run: |
- 7z a -snl llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip .\build\bin\Release\${{ matrix.target }}.dll
+ 7z a -snl llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip .\build\bin\Release\qvac-${{ matrix.target }}.dll
- name: Upload artifacts
uses: actions/upload-artifact@v4
@@ -416,74 +390,6 @@ jobs:
path: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
- windows-cuda:
- runs-on: windows-2022
-
- strategy:
- matrix:
- cuda: ['12.4']
-
- steps:
- - name: Clone
- id: checkout
- uses: actions/checkout@v4
-
- - name: Install ccache
- uses: ggml-org/ccache-action@v1.2.16
- with:
- key: windows-cuda-${{ matrix.cuda }}
- variant: ccache
- evict-old-files: 1d
-
- - name: Install Cuda Toolkit
- uses: ./.github/actions/windows-setup-cuda
- with:
- cuda_version: ${{ matrix.cuda }}
-
- - name: Install Ninja
- id: install_ninja
- run: |
- choco install ninja
-
- - name: Build
- id: cmake_build
- shell: cmd
- run: |
- call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
- cmake -S . -B build -G "Ninja Multi-Config" ^
- -DGGML_BACKEND_DL=ON ^
- -DGGML_NATIVE=OFF ^
- -DGGML_CPU=OFF ^
- -DGGML_CUDA=ON ^
- -DLLAMA_CURL=OFF
- set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
- cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda
-
- - name: Pack artifacts
- id: pack_artifacts
- run: |
- 7z a -snl llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip .\build\bin\Release\ggml-cuda.dll
-
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- path: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
- name: llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
-
- - name: Copy and pack Cuda runtime
- run: |
- echo "Cuda install location: ${{ env.CUDA_PATH }}"
- $dst='.\build\bin\cudart\'
- robocopy "${{env.CUDA_PATH}}\bin" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
- robocopy "${{env.CUDA_PATH}}\lib" $dst cudart64_*.dll cublas64_*.dll cublasLt64_*.dll
- 7z a cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip $dst\*
-
- - name: Upload Cuda runtime
- uses: actions/upload-artifact@v4
- with:
- path: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
- name: cudart-llama-bin-win-cuda-${{ matrix.cuda }}-x64.zip
-
windows-sycl:
runs-on: windows-2022
@@ -780,6 +686,47 @@ jobs:
path: llama-${{ steps.tag.outputs.name }}-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}.tar.gz
name: llama-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}.tar.gz
+ android-build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Set up JDK
+ uses: actions/setup-java@v3
+ with:
+ java-version: 17
+ distribution: zulu
+
+ - name: Setup Android SDK
+ uses: android-actions/setup-android@v3
+ with:
+ log-accepted-android-sdk-licenses: false
+
+ - name: Build
+ run: |
+ cd examples/llama.android
+ ./gradlew build --no-daemon
+
+ - name: Determine tag name
+ id: tag
+ uses: ./.github/actions/get-tag-name
+
+ - name: Pack artifacts
+ id: pack_artifacts
+ run: |
+ cp LICENSE ./examples/llama.android/app/build/
+ zip -r llama-${{ steps.tag.outputs.name }}-bin-android.zip ./examples/llama.android/app/build/*
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ path: llama-${{ steps.tag.outputs.name }}-bin-android.zip
+ name: llama-bin-android.zip
+
release:
if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -793,7 +740,6 @@ jobs:
needs:
- windows
- windows-cpu
- - windows-cuda
- windows-sycl
- windows-hip
- ubuntu-22-cpu
@@ -802,6 +748,7 @@ jobs:
- macOS-x64
- ios-xcode-build
- openEuler-cann
+ - android-build
steps:
- name: Clone
@@ -883,16 +830,17 @@ jobs:
**Linux:**
- [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
- [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
- - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
**Windows:**
- [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip)
- [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
- - [Windows x64 (CUDA)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip)
- [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
- [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
- [Windows x64 (HIP)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-hip-radeon-x64.zip)
+ **Android:**
+ - [Android](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android.zip)
+
**openEuler:**
- [openEuler x86 (310p)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-310p-openEuler-x86.tar.gz)
- [openEuler x86 (910b)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-910b-openEuler-x86.tar.gz)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1085b545a96e..0d6f0c43285e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -104,12 +104,18 @@ option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT})
+# specific extras
+option(LLAMA_MTMD "llama: multimodal support" ${LLAMA_BUILD_TOOLS})
+
# 3rd party libs
option(LLAMA_CURL "llama: use libcurl to download model from an URL" ON)
option(LLAMA_HTTPLIB "llama: if libcurl is disabled, use httplib to download model from an URL" ON)
option(LLAMA_OPENSSL "llama: use openssl to support HTTPS" OFF)
option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
+# profiling
+option(FORCE_GGML_VK_PERF_LOGGER "Force vk performance logging in ggml" OFF)
+
# Required for relocatable CMake package
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
@@ -220,6 +226,10 @@ if (LLAMA_BUILD_COMMON)
endif()
endif()
+if(LLAMA_BUILD_EXAMPLES OR LLAMA_BUILD_TESTS)
+ add_subdirectory(common_test)
+endif()
+
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
include(CTest)
add_subdirectory(tests)
@@ -232,6 +242,8 @@ endif()
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS)
add_subdirectory(tools)
+elseif (LLAMA_MTMD)
+ add_subdirectory(tools/mtmd)
endif()
#
@@ -241,7 +253,7 @@ endif()
include(GNUInstallDirs)
include(CMakePackageConfigHelpers)
-set(LLAMA_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files")
+set(LLAMA_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR}/llama CACHE PATH "Location of header files")
set(LLAMA_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files")
set(LLAMA_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files")
@@ -253,15 +265,52 @@ set_target_properties(llama
PROPERTIES
PUBLIC_HEADER "${LLAMA_PUBLIC_HEADERS}")
-install(TARGETS llama LIBRARY PUBLIC_HEADER)
+install(
+ TARGETS llama
+ EXPORT llama-targets
+ PUBLIC_HEADER
+ DESTINATION ${LLAMA_INCLUDE_INSTALL_DIR})
+
+if (LLAMA_BUILD_COMMON)
+
+ if (NOT LLAMA_CURL AND LLAMA_HTTPLIB)
+ install(
+ TARGETS cpp-httplib
+ EXPORT llama-targets)
+ endif()
+
+ install(
+ TARGETS common build_info
+ EXPORT llama-targets
+ PUBLIC_HEADER
+ DESTINATION ${LLAMA_INCLUDE_INSTALL_DIR}/common)
+
+endif()
+
+if (LLAMA_MTMD AND TARGET mtmd)
+
+ install(
+ TARGETS mtmd
+ EXPORT llama-targets
+ PUBLIC_HEADER
+ DESTINATION ${LLAMA_INCLUDE_INSTALL_DIR}/mtmd)
+
+endif()
+
+install(
+ EXPORT llama-targets
+ FILE llama-targets.cmake
+ NAMESPACE llama::
+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/llama)
+
+install(
+ FILES ${CMAKE_CURRENT_BINARY_DIR}/llama-config.cmake
+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/llama)
configure_package_config_file(
- ${CMAKE_CURRENT_SOURCE_DIR}/cmake/llama-config.cmake.in
- ${CMAKE_CURRENT_BINARY_DIR}/llama-config.cmake
- INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/llama
- PATH_VARS LLAMA_INCLUDE_INSTALL_DIR
- LLAMA_LIB_INSTALL_DIR
- LLAMA_BIN_INSTALL_DIR )
+ ${CMAKE_CURRENT_SOURCE_DIR}/cmake/llama-config.cmake.in
+ ${CMAKE_CURRENT_BINARY_DIR}/llama-config.cmake
+ INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/llama)
write_basic_package_version_file(
${CMAKE_CURRENT_BINARY_DIR}/llama-version.cmake
@@ -270,7 +319,7 @@ write_basic_package_version_file(
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/llama-config.cmake
${CMAKE_CURRENT_BINARY_DIR}/llama-version.cmake
- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/llama)
+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/llama)
install(
FILES convert_hf_to_gguf.py
@@ -284,6 +333,10 @@ install(
WORLD_EXECUTE
DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(
+ PROGRAMS vulkan_profiling_analyzer.py
+ DESTINATION ${CMAKE_INSTALL_BINDIR})
+
configure_file(cmake/llama.pc.in
"${CMAKE_CURRENT_BINARY_DIR}/llama.pc"
@ONLY)
diff --git a/README.md b/README.md
index 2e44ae7d0c79..077454f781d3 100644
--- a/README.md
+++ b/README.md
@@ -1,608 +1,156 @@
-# llama.cpp
+# qvac-fabric-llm.cpp
-
+**AI inference and training engine for desktop and mobile platforms.**
[](https://opensource.org/licenses/MIT)
-[](https://github.com/ggml-org/llama.cpp/releases)
-[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
+[](https://github.com/ggml-org/llama.cpp)
-[Manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md)
+`qvac-fabric-llm.cpp` is a specialized fork of [llama.cpp](https://github.com/ggml-org/llama.cpp) optimized for embedded systems, mobile devices, and enterprise deployment scenarios. It extends the excellent foundation of llama.cpp with additional capabilities focused on memory-based model loading, mobile GPU optimization, and flexible integration patterns.
-LLM inference in C/C++
-## Recent API changes
+## Key Features
-- [Changelog for `libllama` API](https://github.com/ggml-org/llama.cpp/issues/9289)
-- [Changelog for `llama-server` REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
+The following capabilities are developed and maintained as part of qvac-fabric-llm.cpp. Features marked as *exclusive* are not available in upstream llama.cpp.
-## Hot topics
+### LoRA Fine-Tuning *(exclusive)*
-- **[guide : using the new WebUI of llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/16938)**
-- [guide : running gpt-oss with llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/15396)
-- [[FEEDBACK] Better packaging for llama.cpp to support downstream consumers 🤗](https://github.com/ggml-org/llama.cpp/discussions/15313)
-- Support for the `gpt-oss` model with native MXFP4 format has been added | [PR](https://github.com/ggml-org/llama.cpp/pull/15091) | [Collaboration with NVIDIA](https://blogs.nvidia.com/blog/rtx-ai-garage-openai-oss) | [Comment](https://github.com/ggml-org/llama.cpp/discussions/15095)
-- Multimodal support arrived in `llama-server`: [#12898](https://github.com/ggml-org/llama.cpp/pull/12898) | [documentation](./docs/multimodal.md)
-- VS Code extension for FIM completions: https://github.com/ggml-org/llama.vscode
-- Vim/Neovim plugin for FIM completions: https://github.com/ggml-org/llama.vim
-- Hugging Face Inference Endpoints now support GGUF out of the box! https://github.com/ggml-org/llama.cpp/discussions/9669
-- Hugging Face GGUF editor: [discussion](https://github.com/ggml-org/llama.cpp/discussions/9268) | [tool](https://huggingface.co/spaces/CISCai/gguf-editor)
+`qvac-fabric-llm.cpp` provides native [LoRA](https://arxiv.org/abs/2106.09685) (Low-Rank Adaptation) fine-tuning across CPU, Vulkan, and Metal backends. The training pipeline runs directly on consumer hardware, including mobile phones and integrated GPUs.
-----
+- Multi-backend GPU training (NVIDIA, AMD, Intel, Apple, Mali, Adreno)
+- FP32, FP16, Q8, and Q4 training paths
+- Supervised instruction-tuning via assistant-only masked loss (SFT)
+- Checkpoint saving and resumable training
+- Learning rate schedulers (constant, cosine, linear) with warmup support
+- LoRA adapter merging into base models via **llama-export-lora**
+- Verified compatibility with Qwen3 and Gemma3 model architectures
-## Quick start
+For usage details and CLI reference, see the [Finetuning Guide](examples/training/README.md).
-Getting started with llama.cpp is straightforward. Here are several ways to install it on your machine:
+### BitNet Inference and Fine-Tuning *(exclusive)*
-- Install `llama.cpp` using [brew, nix or winget](docs/install.md)
-- Run with Docker - see our [Docker documentation](docs/docker.md)
-- Download pre-built binaries from the [releases page](https://github.com/ggml-org/llama.cpp/releases)
-- Build from source by cloning this repository - check out [our build guide](docs/build.md)
+Native support for [BitNet](https://arxiv.org/abs/2402.17764) ternary quantized models via the TQ2_0 data type, enabling efficient inference and LoRA fine-tuning of models such as [bitnet_b1_58-xl](https://huggingface.co/qvac/fabric-llm-finetune-bitnet) on resource-constrained devices.
-Once installed, you'll need a model to work with. Head to the [Obtaining and quantizing models](#obtaining-and-quantizing-models) section to learn more.
+The official [microsoft/BitNet](https://github.com/microsoft/BitNet) inference framework provides optimized CPU kernels and GPU support limited to CUDA. `qvac-fabric-llm.cpp` **extends** BitNet to all major GPU backends -- Vulkan, Metal, and CPU -- bringing cross-platform GPU-accelerated BitNet inference and on-device fine-tuning to hardware not covered by the upstream framework, including Apple Silicon, mobile GPUs (Adreno, Mali), and AMD/Intel discrete GPUs. Compatible with models such as [bitnet_b1_58-3B](https://huggingface.co/1bitLLM/bitnet_b1_58-3B).
-Example command:
+- **Backends**: Vulkan, Metal and CPU TQ2_0 quantization support
+- **Training**: LoRA fine-tuning of BitNet models on Vulkan, Metal, and CPU backends
+- **Conversion**: HuggingFace-to-GGUF conversion for BitNet model architectures
+- Cooperative matrix (coopmat) support for Vulkan devices that expose the extension
-```sh
-# Use a local model file
-llama-cli -m my_model.gguf
+### Memory-Based Model Loading *(exclusive)*
-# Or download and run a model directly from Hugging Face
-llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
+Load models directly from memory buffers instead of the filesystem, enabling deployment in environments where disk access is restricted or unavailable.
-# Launch OpenAI-compatible API server
-llama-server -hf ggml-org/gemma-3-1b-it-GGUF
-```
-
-## Description
-
-The main goal of `llama.cpp` is to enable LLM inference with minimal setup and state-of-the-art performance on a wide
-range of hardware - locally and in the cloud.
-
-- Plain C/C++ implementation without any dependencies
-- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
-- AVX, AVX2, AVX512 and AMX support for x86 architectures
-- RVV, ZVFH, ZFH and ZICBOP support for RISC-V architectures
-- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
-- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
-- Vulkan and SYCL backend support
-- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
-
-The `llama.cpp` project is the main playground for developing new features for the [ggml](https://github.com/ggml-org/ggml) library.
-
-
-Models
-
-Typically finetunes of the base models below are supported as well.
-
-Instructions for adding support for new models: [HOWTO-add-model.md](docs/development/HOWTO-add-model.md)
-
-#### Text-only
-
-- [X] LLaMA 🦙
-- [x] LLaMA 2 🦙🦙
-- [x] LLaMA 3 🦙🦙🦙
-- [X] [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1)
-- [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral)
-- [x] [DBRX](https://huggingface.co/databricks/dbrx-instruct)
-- [x] [Jamba](https://huggingface.co/ai21labs)
-- [X] [Falcon](https://huggingface.co/models?search=tiiuae/falcon)
-- [X] [Chinese LLaMA / Alpaca](https://github.com/ymcui/Chinese-LLaMA-Alpaca) and [Chinese LLaMA-2 / Alpaca-2](https://github.com/ymcui/Chinese-LLaMA-Alpaca-2)
-- [X] [Vigogne (French)](https://github.com/bofenghuang/vigogne)
-- [X] [BERT](https://github.com/ggml-org/llama.cpp/pull/5423)
-- [X] [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
-- [X] [Baichuan 1 & 2](https://huggingface.co/models?search=baichuan-inc/Baichuan) + [derivations](https://huggingface.co/hiyouga/baichuan-7b-sft)
-- [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila)
-- [X] [Starcoder models](https://github.com/ggml-org/llama.cpp/pull/3187)
-- [X] [Refact](https://huggingface.co/smallcloudai/Refact-1_6B-fim)
-- [X] [MPT](https://github.com/ggml-org/llama.cpp/pull/3417)
-- [X] [Bloom](https://github.com/ggml-org/llama.cpp/pull/3553)
-- [x] [Yi models](https://huggingface.co/models?search=01-ai/Yi)
-- [X] [StableLM models](https://huggingface.co/stabilityai)
-- [x] [Deepseek models](https://huggingface.co/models?search=deepseek-ai/deepseek)
-- [x] [Qwen models](https://huggingface.co/models?search=Qwen/Qwen)
-- [x] [PLaMo-13B](https://github.com/ggml-org/llama.cpp/pull/3557)
-- [x] [Phi models](https://huggingface.co/models?search=microsoft/phi)
-- [x] [PhiMoE](https://github.com/ggml-org/llama.cpp/pull/11003)
-- [x] [GPT-2](https://huggingface.co/gpt2)
-- [x] [Orion 14B](https://github.com/ggml-org/llama.cpp/pull/5118)
-- [x] [InternLM2](https://huggingface.co/models?search=internlm2)
-- [x] [CodeShell](https://github.com/WisdomShell/codeshell)
-- [x] [Gemma](https://ai.google.dev/gemma)
-- [x] [Mamba](https://github.com/state-spaces/mamba)
-- [x] [Grok-1](https://huggingface.co/keyfan/grok-1-hf)
-- [x] [Xverse](https://huggingface.co/models?search=xverse)
-- [x] [Command-R models](https://huggingface.co/models?search=CohereForAI/c4ai-command-r)
-- [x] [SEA-LION](https://huggingface.co/models?search=sea-lion)
-- [x] [GritLM-7B](https://huggingface.co/GritLM/GritLM-7B) + [GritLM-8x7B](https://huggingface.co/GritLM/GritLM-8x7B)
-- [x] [OLMo](https://allenai.org/olmo)
-- [x] [OLMo 2](https://allenai.org/olmo)
-- [x] [OLMoE](https://huggingface.co/allenai/OLMoE-1B-7B-0924)
-- [x] [Granite models](https://huggingface.co/collections/ibm-granite/granite-code-models-6624c5cec322e4c148c8b330)
-- [x] [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) + [Pythia](https://github.com/EleutherAI/pythia)
-- [x] [Snowflake-Arctic MoE](https://huggingface.co/collections/Snowflake/arctic-66290090abe542894a5ac520)
-- [x] [Smaug](https://huggingface.co/models?search=Smaug)
-- [x] [Poro 34B](https://huggingface.co/LumiOpen/Poro-34B)
-- [x] [Bitnet b1.58 models](https://huggingface.co/1bitLLM)
-- [x] [Flan T5](https://huggingface.co/models?search=flan-t5)
-- [x] [Open Elm models](https://huggingface.co/collections/apple/openelm-instruct-models-6619ad295d7ae9f868b759ca)
-- [x] [ChatGLM3-6b](https://huggingface.co/THUDM/chatglm3-6b) + [ChatGLM4-9b](https://huggingface.co/THUDM/glm-4-9b) + [GLMEdge-1.5b](https://huggingface.co/THUDM/glm-edge-1.5b-chat) + [GLMEdge-4b](https://huggingface.co/THUDM/glm-edge-4b-chat)
-- [x] [GLM-4-0414](https://huggingface.co/collections/THUDM/glm-4-0414-67f3cbcb34dd9d252707cb2e)
-- [x] [SmolLM](https://huggingface.co/collections/HuggingFaceTB/smollm-6695016cad7167254ce15966)
-- [x] [EXAONE-3.0-7.8B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct)
-- [x] [FalconMamba Models](https://huggingface.co/collections/tiiuae/falconmamba-7b-66b9a580324dd1598b0f6d4a)
-- [x] [Jais](https://huggingface.co/inceptionai/jais-13b-chat)
-- [x] [Bielik-11B-v2.3](https://huggingface.co/collections/speakleash/bielik-11b-v23-66ee813238d9b526a072408a)
-- [x] [RWKV-6](https://github.com/BlinkDL/RWKV-LM)
-- [x] [QRWKV-6](https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1)
-- [x] [GigaChat-20B-A3B](https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct)
-- [X] [Trillion-7B-preview](https://huggingface.co/trillionlabs/Trillion-7B-preview)
-- [x] [Ling models](https://huggingface.co/collections/inclusionAI/ling-67c51c85b34a7ea0aba94c32)
-- [x] [LFM2 models](https://huggingface.co/collections/LiquidAI/lfm2-686d721927015b2ad73eaa38)
-- [x] [Hunyuan models](https://huggingface.co/collections/tencent/hunyuan-dense-model-6890632cda26b19119c9c5e7)
-- [x] [BailingMoeV2 (Ring/Ling 2.0) models](https://huggingface.co/collections/inclusionAI/ling-v2-68bf1dd2fc34c306c1fa6f86)
-
-#### Multimodal
-
-- [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e), [LLaVA 1.6 models](https://huggingface.co/collections/liuhaotian/llava-16-65b9e40155f60fd046a5ccf2)
-- [x] [BakLLaVA](https://huggingface.co/models?search=SkunkworksAI/Bakllava)
-- [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5)
-- [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V)
-- [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM)
-- [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL)
-- [x] [Mini CPM](https://huggingface.co/models?search=MiniCPM)
-- [x] [Moondream](https://huggingface.co/vikhyatk/moondream2)
-- [x] [Bunny](https://github.com/BAAI-DCAI/Bunny)
-- [x] [GLM-EDGE](https://huggingface.co/models?search=glm-edge)
-- [x] [Qwen2-VL](https://huggingface.co/collections/Qwen/qwen2-vl-66cee7455501d7126940800d)
-- [x] [LFM2-VL](https://huggingface.co/collections/LiquidAI/lfm2-vl-68963bbc84a610f7638d5ffa)
-
-
-
-
-Bindings
-
-- Python: [ddh0/easy-llama](https://github.com/ddh0/easy-llama)
-- Python: [abetlen/llama-cpp-python](https://github.com/abetlen/llama-cpp-python)
-- Go: [go-skynet/go-llama.cpp](https://github.com/go-skynet/go-llama.cpp)
-- Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp)
-- JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp)
-- JS/TS (Programmable Prompt Engine CLI): [offline-ai/cli](https://github.com/offline-ai/cli)
-- JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm)
-- Typescript/Wasm (nicer API, available on npm): [ngxson/wllama](https://github.com/ngxson/wllama)
-- Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb)
-- Rust (more features): [edgenai/llama_cpp-rs](https://github.com/edgenai/llama_cpp-rs)
-- Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp)
-- Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs)
-- Rust (automated build from crates.io): [ShelbyJenkins/llm_client](https://github.com/ShelbyJenkins/llm_client)
-- C#/.NET: [SciSharp/LLamaSharp](https://github.com/SciSharp/LLamaSharp)
-- C#/VB.NET (more features - community license): [LM-Kit.NET](https://docs.lm-kit.com/lm-kit-net/index.html)
-- Scala 3: [donderom/llm4s](https://github.com/donderom/llm4s)
-- Clojure: [phronmophobic/llama.clj](https://github.com/phronmophobic/llama.clj)
-- React Native: [mybigday/llama.rn](https://github.com/mybigday/llama.rn)
-- Java: [kherud/java-llama.cpp](https://github.com/kherud/java-llama.cpp)
-- Java: [QuasarByte/llama-cpp-jna](https://github.com/QuasarByte/llama-cpp-jna)
-- Zig: [deins/llama.cpp.zig](https://github.com/Deins/llama.cpp.zig)
-- Flutter/Dart: [netdur/llama_cpp_dart](https://github.com/netdur/llama_cpp_dart)
-- Flutter: [xuegao-tzx/Fllama](https://github.com/xuegao-tzx/Fllama)
-- PHP (API bindings and features built on top of llama.cpp): [distantmagic/resonance](https://github.com/distantmagic/resonance) [(more info)](https://github.com/ggml-org/llama.cpp/pull/6326)
-- Guile Scheme: [guile_llama_cpp](https://savannah.nongnu.org/projects/guile-llama-cpp)
-- Swift [srgtuszy/llama-cpp-swift](https://github.com/srgtuszy/llama-cpp-swift)
-- Swift [ShenghaiWang/SwiftLlama](https://github.com/ShenghaiWang/SwiftLlama)
-- Delphi [Embarcadero/llama-cpp-delphi](https://github.com/Embarcadero/llama-cpp-delphi)
-- Go (no CGo needed): [hybridgroup/yzma](https://github.com/hybridgroup/yzma)
-
-
-
-
-UIs
-
-*(to have a project listed here, it should clearly state that it depends on `llama.cpp`)*
-
-- [AI Sublime Text plugin](https://github.com/yaroslavyaroslav/OpenAI-sublime-text) (MIT)
-- [cztomsik/ava](https://github.com/cztomsik/ava) (MIT)
-- [Dot](https://github.com/alexpinel/Dot) (GPL)
-- [eva](https://github.com/ylsdamxssjxxdd/eva) (MIT)
-- [iohub/collama](https://github.com/iohub/coLLaMA) (Apache-2.0)
-- [janhq/jan](https://github.com/janhq/jan) (AGPL)
-- [johnbean393/Sidekick](https://github.com/johnbean393/Sidekick) (MIT)
-- [KanTV](https://github.com/zhouwg/kantv?tab=readme-ov-file) (Apache-2.0)
-- [KodiBot](https://github.com/firatkiral/kodibot) (GPL)
-- [llama.vim](https://github.com/ggml-org/llama.vim) (MIT)
-- [LARS](https://github.com/abgulati/LARS) (AGPL)
-- [Llama Assistant](https://github.com/vietanhdev/llama-assistant) (GPL)
-- [LLMFarm](https://github.com/guinmoon/LLMFarm?tab=readme-ov-file) (MIT)
-- [LLMUnity](https://github.com/undreamai/LLMUnity) (MIT)
-- [LMStudio](https://lmstudio.ai/) (proprietary)
-- [LocalAI](https://github.com/mudler/LocalAI) (MIT)
-- [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL)
-- [MindMac](https://mindmac.app) (proprietary)
-- [MindWorkAI/AI-Studio](https://github.com/MindWorkAI/AI-Studio) (FSL-1.1-MIT)
-- [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT)
-- [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) (Apache-2.0)
-- [nat/openplayground](https://github.com/nat/openplayground) (MIT)
-- [nomic-ai/gpt4all](https://github.com/nomic-ai/gpt4all) (MIT)
-- [ollama/ollama](https://github.com/ollama/ollama) (MIT)
-- [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) (AGPL)
-- [PocketPal AI](https://github.com/a-ghorbani/pocketpal-ai) (MIT)
-- [psugihara/FreeChat](https://github.com/psugihara/FreeChat) (MIT)
-- [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) (MIT)
-- [pythops/tenere](https://github.com/pythops/tenere) (AGPL)
-- [ramalama](https://github.com/containers/ramalama) (MIT)
-- [semperai/amica](https://github.com/semperai/amica) (MIT)
-- [withcatai/catai](https://github.com/withcatai/catai) (MIT)
-- [Autopen](https://github.com/blackhole89/autopen) (GPL)
-
-
-
-
-Tools
-
-- [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from HuggingFace Hub and convert them to GGML
-- [akx/ollama-dl](https://github.com/akx/ollama-dl) – download models from the Ollama library to be used directly with llama.cpp
-- [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption
-- [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage
-- [Styled Lines](https://marketplace.unity.com/packages/tools/generative-ai/styled-lines-llama-cpp-model-292902) (proprietary licensed, async wrapper of inference part for game development in Unity3d with pre-built Mobile and Web platform wrappers and a model example)
-- [unslothai/unsloth](https://github.com/unslothai/unsloth) – 🦥 exports/saves fine-tuned and trained models to GGUF (Apache-2.0)
-
-
-
-
-Infrastructure
-
-- [Paddler](https://github.com/intentee/paddler) - Open-source LLMOps platform for hosting and scaling AI in your own infrastructure
-- [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs
-- [llama_cpp_canister](https://github.com/onicai/llama_cpp_canister) - llama.cpp as a smart contract on the Internet Computer, using WebAssembly
-- [llama-swap](https://github.com/mostlygeek/llama-swap) - transparent proxy that adds automatic model switching with llama-server
-- [Kalavai](https://github.com/kalavai-net/kalavai-client) - Crowdsource end to end LLM deployment at any scale
-- [llmaz](https://github.com/InftyAI/llmaz) - ☸️ Easy, advanced inference platform for large language models on Kubernetes.
-
-
-
-Games
-
-- [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you.
-
-
-
-
-## Supported backends
-
-| Backend | Target devices |
-| --- | --- |
-| [Metal](docs/build.md#metal-build) | Apple Silicon |
-| [BLAS](docs/build.md#blas-build) | All |
-| [BLIS](docs/backend/BLIS.md) | All |
-| [SYCL](docs/backend/SYCL.md) | Intel and Nvidia GPU |
-| [MUSA](docs/build.md#musa) | Moore Threads GPU |
-| [CUDA](docs/build.md#cuda) | Nvidia GPU |
-| [HIP](docs/build.md#hip) | AMD GPU |
-| [Vulkan](docs/build.md#vulkan) | GPU |
-| [CANN](docs/build.md#cann) | Ascend NPU |
-| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
-| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
-| [WebGPU [In Progress]](docs/build.md#webgpu) | All |
-| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
-| [Hexagon [In Progress]](docs/backend/hexagon/README.md) | Snapdragon |
-
-## Obtaining and quantizing models
-
-The [Hugging Face](https://huggingface.co) platform hosts a [number of LLMs](https://huggingface.co/models?library=gguf&sort=trending) compatible with `llama.cpp`:
-
-- [Trending](https://huggingface.co/models?library=gguf&sort=trending)
-- [LLaMA](https://huggingface.co/models?sort=trending&search=llama+gguf)
-
-You can either manually download the GGUF file or directly use any `llama.cpp`-compatible models from [Hugging Face](https://huggingface.co/) or other model hosting sites, such as [ModelScope](https://modelscope.cn/), by using this CLI argument: `-hf /[:quant]`. For example:
-
-```sh
-llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
-```
-
-By default, the CLI would download from Hugging Face, you can switch to other options with the environment variable `MODEL_ENDPOINT`. For example, you may opt to downloading model checkpoints from ModelScope or other model sharing communities by setting the environment variable, e.g. `MODEL_ENDPOINT=https://www.modelscope.cn/`.
-
-After downloading a model, use the CLI tools to run it locally - see below.
-
-`llama.cpp` requires the model to be stored in the [GGUF](https://github.com/ggml-org/ggml/blob/master/docs/gguf.md) file format. Models in other data formats can be converted to GGUF using the `convert_*.py` Python scripts in this repo.
-
-The Hugging Face platform provides a variety of online tools for converting, quantizing and hosting models with `llama.cpp`:
-
-- Use the [GGUF-my-repo space](https://huggingface.co/spaces/ggml-org/gguf-my-repo) to convert to GGUF format and quantize model weights to smaller sizes
-- Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123)
-- Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268)
-- Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669)
-
-To learn more about model quantization, [read this documentation](tools/quantize/README.md)
-
-## [`llama-cli`](tools/main)
-
-#### A CLI tool for accessing and experimenting with most of `llama.cpp`'s functionality.
-
--
- Run in conversation mode
-
- Models with a built-in chat template will automatically activate conversation mode. If this doesn't occur, you can manually enable it by adding `-cnv` and specifying a suitable chat template with `--chat-template NAME`
-
- ```bash
- llama-cli -m model.gguf
-
- # > hi, who are you?
- # Hi there! I'm your helpful assistant! I'm an AI-powered chatbot designed to assist and provide information to users like you. I'm here to help answer your questions, provide guidance, and offer support on a wide range of topics. I'm a friendly and knowledgeable AI, and I'm always happy to help with anything you need. What's on your mind, and how can I assist you today?
- #
- # > what is 1+1?
- # Easy peasy! The answer to 1+1 is... 2!
- ```
-
-
-
--
- Run in conversation mode with custom chat template
-
- ```bash
- # use the "chatml" template (use -h to see the list of supported templates)
- llama-cli -m model.gguf -cnv --chat-template chatml
-
- # use a custom template
- llama-cli -m model.gguf -cnv --in-prefix 'User: ' --reverse-prompt 'User:'
- ```
-
-
-
--
- Run simple text completion
-
- To disable conversation mode explicitly, use `-no-cnv`
-
- ```bash
- llama-cli -m model.gguf -p "I believe the meaning of life is" -n 128 -no-cnv
-
- # I believe the meaning of life is to find your own truth and to live in accordance with it. For me, this means being true to myself and following my passions, even if they don't align with societal expectations. I think that's what I love about yoga – it's not just a physical practice, but a spiritual one too. It's about connecting with yourself, listening to your inner voice, and honoring your own unique journey.
- ```
-
-
-
--
- Constrain the output with a custom grammar
-
- ```bash
- llama-cli -m model.gguf -n 256 --grammar-file grammars/json.gbnf -p 'Request: schedule a call at 8pm; Command:'
-
- # {"appointmentTime": "8pm", "appointmentDetails": "schedule a a call"}
- ```
-
- The [grammars/](grammars/) folder contains a handful of sample grammars. To write your own, check out the [GBNF Guide](grammars/README.md).
-
- For authoring more complex JSON grammars, check out https://grammar.intrinsiclabs.ai/
+- Embedded systems with limited or no filesystem access
+- WebAssembly deployments where models are fetched over the network
+- Encrypted model storage where models are decrypted in memory
+- Streaming scenarios where models arrive over network connections
-
+```cpp
+#include "llama-cpp.h"
+std::vector model_data = /* load from network, decrypt, etc. */;
+auto model = llama_model_load_from_buffer(std::move(model_data), params);
-## [`llama-server`](tools/server)
-
-#### A lightweight, [OpenAI API](https://github.com/openai/openai-openapi) compatible, HTTP server for serving LLMs.
-
--
- Start a local HTTP server with default configuration on port 8080
-
- ```bash
- llama-server -m model.gguf --port 8080
-
- # Basic web UI can be accessed via browser: http://localhost:8080
- # Chat completion endpoint: http://localhost:8080/v1/chat/completions
- ```
-
-
-
--
- Support multiple-users and parallel decoding
-
- ```bash
- # up to 4 concurrent requests, each with 4096 max context
- llama-server -m model.gguf -c 16384 -np 4
- ```
-
-
-
--
- Enable speculative decoding
-
- ```bash
- # the draft.gguf model should be a small variant of the target model.gguf
- llama-server -m model.gguf -md draft.gguf
- ```
-
-
-
--
- Serve an embedding model
-
- ```bash
- # use the /embedding endpoint
- llama-server -m model.gguf --embedding --pooling cls -ub 8192
- ```
-
-
-
--
- Serve a reranking model
-
- ```bash
- # use the /reranking endpoint
- llama-server -m model.gguf --reranking
- ```
-
-
-
--
- Constrain all outputs with a grammar
-
- ```bash
- # custom grammar
- llama-server -m model.gguf --grammar-file grammar.gbnf
+auto model = llama_model_load_from_split_futures(paths, n_paths, context, tensor_list, params);
+llama_model_load_fulfill_split_future(path, context, std::move(streambuf));
+```
- # JSON
- llama-server -m model.gguf --grammar-file grammars/json.gbnf
- ```
+### Mobile GPU Optimization *(exclusive)*
-
+Enhanced GPU support with targeted optimizations for Qualcomm Adreno GPUs.
+- **Vulkan on Adreno 800+**: quantized inference (Q4_0, Q8) and LoRA fine-tuning for Gemma3, Qwen3, and BitNet (TQ2_0) model architectures.
+- **OpenCL on Adreno**: inference support for all other model architectures.
+- Adreno-specific Vulkan shader variants for improved throughput.
+- Vulkan Memory Allocator (VMA) integration for efficient GPU memory management.
-## [`llama-perplexity`](tools/perplexity)
-#### A tool for measuring the [perplexity](tools/perplexity/README.md) [^1] (and other quality metrics) of a model over a given text.
+## Quick Start
--
- Measure the perplexity over a text file
+### Building from Source
- ```bash
- llama-perplexity -m model.gguf -f file.txt
+```bash
+git clone https://github.com/tetherto/qvac-fabric-llm.cpp.git
+cd qvac-fabric-llm.cpp
- # [1]15.2701,[2]5.4007,[3]5.3073,[4]6.2965,[5]5.8940,[6]5.6096,[7]5.7942,[8]4.9297, ...
- # Final estimate: PPL = 5.4007 +/- 0.67339
- ```
+# Standard build
+cmake -B build
+cmake --build build --config Release
-
+# With Vulkan support (Android, Windows, Linux)
+cmake -B build -DGGML_VULKAN=ON
+cmake --build build --config Release
--
- Measure KL divergence
+# With Metal support (macOS, iOS)
+cmake -B build -DGGML_METAL=ON
+cmake --build build --config Release
+```
- ```bash
- # TODO
- ```
+For more detailed build instructions, see [docs/build.md](docs/build.md).
-
+### Running a Model
-[^1]: [https://huggingface.co/docs/transformers/perplexity](https://huggingface.co/docs/transformers/perplexity)
+```bash
+# Run a model with Vulkan GPU acceleration, 4096 context length, and a prompt
+./build/bin/llama-cli -m model.gguf -ngl 99 -c 4096 -p "Explain quantum computing in simple terms"
+```
-## [`llama-bench`](tools/llama-bench)
-#### Benchmark the performance of the inference for various parameters.
+## Supported Platforms
--
- Run default benchmark
+| Platform | Backend | Status |
+|----------|---------|--------|
+| Linux (x86_64, ARM64) | CPU, Vulkan, CUDA | ✅ Full support |
+| macOS (Intel, Apple Silicon) | CPU, Metal | ✅ Full support |
+| Windows (x86_64) | CPU, Vulkan, CUDA | ✅ Full support |
+| Android (ARM64) | CPU, Vulkan, OpenCL | ✅ Full support |
+| iOS | CPU, Metal | ✅ Full support |
- ```bash
- llama-bench -m model.gguf
- # Output:
- # | model | size | params | backend | threads | test | t/s |
- # | ------------------- | ---------: | ---------: | ---------- | ------: | ------------: | -------------------: |
- # | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | pp512 | 5765.41 ± 20.55 |
- # | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | tg128 | 197.71 ± 0.81 |
- #
- # build: 3e0ba0e60 (4229)
- ```
+## Relationship with llama.cpp
-
+qvac-fabric-llm.cpp is a maintained fork of [llama.cpp](https://github.com/ggml-org/llama.cpp). The project regularly synchronizes with upstream releases to incorporate improvements, bug fixes, and new model support, while extending the engine with capabilities not present in the upstream project.
-## [`llama-run`](tools/run)
+**Current upstream baseline:** llama.cpp b7248
-#### A comprehensive example for running `llama.cpp` models. Useful for inferencing. Used with RamaLama [^3].
+### Exclusive Features
--
- Run a model with a specific prompt (by default it's pulled from Ollama registry)
+The following features are developed in qvac-fabric-llm.cpp and are not available in upstream llama.cpp:
- ```bash
- llama-run granite-code
- ```
+| Feature | Description |
+|---------|-------------|
+| LoRA fine-tuning | On-device training across CPU, Vulkan, and Metal with SFT, checkpointing, and LR scheduling |
+| BitNet inference and training | TQ2_0 quantization on Vulkan, Metal, and CPU for inference and LoRA fine-tuning; extends [microsoft/BitNet](https://github.com/microsoft/BitNet) beyond its CUDA-only GPU support |
+| Memory-based model loading | Load models from in-memory buffers with split-model and async fulfillment support |
+| Mobile GPU optimization | Adreno 800+ quantized inference (Q4_0, Q8), Adreno-specific Vulkan shader variants, VMA integration |
-
+### Upstream Compatibility
-[^3]: [RamaLama](https://github.com/containers/ramalama)
+All standard llama.cpp functionality, models, and APIs remain fully compatible.
-## [`llama-simple`](examples/simple)
+- Any GGUF model supported by llama.cpp is supported by qvac-fabric-llm.cpp
+- Existing llama.cpp documentation applies to all non-exclusive features
-#### A minimal example for implementing apps with `llama.cpp`. Useful for developers.
--
- Basic text completion
+## Contributing
- ```bash
- llama-simple -m model.gguf
+We welcome contributions! Please see our development workflow:
- # Hello my name is Kaitlyn and I am a 16 year old girl. I am a junior in high school and I am currently taking a class called "The Art of
- ```
+1. Fork the repository
+2. Create a feature branch from `master`
+3. Submit a pull request
-
+## License
-## Contributing
+MIT License - see [LICENSE](LICENSE) for details.
-- Contributors can open PRs
-- Collaborators will be invited based on contributions
-- Maintainers can push to branches in the `llama.cpp` repo and merge PRs into the `master` branch
-- Any help with managing issues, PRs and projects is very appreciated!
-- See [good first issues](https://github.com/ggml-org/llama.cpp/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) for tasks suitable for first contributions
-- Read the [CONTRIBUTING.md](CONTRIBUTING.md) for more information
-- Make sure to read this: [Inference at the edge](https://github.com/ggml-org/llama.cpp/discussions/205)
-- A bit of backstory for those who are interested: [Changelog podcast](https://changelog.com/podcast/532)
-
-## Other documentation
-
-- [main (cli)](tools/main/README.md)
-- [server](tools/server/README.md)
-- [GBNF grammars](grammars/README.md)
-
-#### Development documentation
-
-- [How to build](docs/build.md)
-- [Running on Docker](docs/docker.md)
-- [Build on Android](docs/android.md)
-- [Performance troubleshooting](docs/development/token_generation_performance_tips.md)
-- [GGML tips & tricks](https://github.com/ggml-org/llama.cpp/wiki/GGML-Tips-&-Tricks)
-
-#### Seminal papers and background on the models
-
-If your issue is with model generation quality, then please at least scan the following links and papers to understand the limitations of LLaMA models. This is especially important when choosing an appropriate model size and appreciating both the significant and subtle differences between LLaMA models and ChatGPT:
-- LLaMA:
- - [Introducing LLaMA: A foundational, 65-billion-parameter large language model](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/)
- - [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971)
-- GPT-3
- - [Language Models are Few-Shot Learners](https://arxiv.org/abs/2005.14165)
-- GPT-3.5 / InstructGPT / ChatGPT:
- - [Aligning language models to follow instructions](https://openai.com/research/instruction-following)
- - [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155)
-
-## XCFramework
-The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS,
-and macOS. It can be used in Swift projects without the need to compile the
-library from source. For example:
-```swift
-// swift-tools-version: 5.10
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-
-import PackageDescription
-
-let package = Package(
- name: "MyLlamaPackage",
- targets: [
- .executableTarget(
- name: "MyLlamaPackage",
- dependencies: [
- "LlamaFramework"
- ]),
- .binaryTarget(
- name: "LlamaFramework",
- url: "https://github.com/ggml-org/llama.cpp/releases/download/b5046/llama-b5046-xcframework.zip",
- checksum: "c19be78b5f00d8d29a25da41042cb7afa094cbf6280a225abe614b03b20029ab"
- )
- ]
-)
-```
-The above example is using an intermediate build `b5046` of the library. This can be modified
-to use a different version by changing the URL and checksum.
+qvac-fabric-llm.cpp is built on [llama.cpp](https://github.com/ggml-org/llama.cpp) by Georgi Gerganov and contributors.
-## Completions
-Command-line completion is available for some environments.
+---
-#### Bash Completion
-```bash
-$ build/bin/llama-cli --completion-bash > ~/.llama-completion.bash
-$ source ~/.llama-completion.bash
-```
-Optionally this can be added to your `.bashrc` or `.bash_profile` to load it
-automatically. For example:
-```console
-$ echo "source ~/.llama-completion.bash" >> ~/.bashrc
-```
+For additional documentation, refer to the [llama.cpp documentation](https://github.com/ggml-org/llama.cpp/tree/master/docs).
## Dependencies
@@ -613,4 +161,3 @@ $ echo "source ~/.llama-completion.bash" >> ~/.bashrc
- [linenoise.cpp](./tools/run/linenoise.cpp/linenoise.cpp) - C++ library that provides readline-like line editing capabilities, used by `llama-run` - BSD 2-Clause License
- [curl](https://curl.se/) - Client-side URL transfer library, used by various tools/examples - [CURL License](https://curl.se/docs/copyright.html)
- [miniaudio.h](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
-- [subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
diff --git a/ci/run.sh b/ci/run.sh
index 83b2603e8210..7fa469b14b57 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -236,6 +236,11 @@ function gg_run_ctest_release {
# Check cmake, make and ctest are installed
gg_check_build_requirements
+ # Disable native CPU optimizations for low-perf builds to ensure compatibility
+ if [ ! -z ${GG_BUILD_LOW_PERF} ]; then
+ CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF"
+ fi
+
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
diff --git a/cmake/llama-config.cmake.in b/cmake/llama-config.cmake.in
index 90cbec5b6f13..5a1d6a8a33b8 100644
--- a/cmake/llama-config.cmake.in
+++ b/cmake/llama-config.cmake.in
@@ -5,26 +5,9 @@ set(LLAMA_SHARED_LIB @BUILD_SHARED_LIBS@)
@PACKAGE_INIT@
-set_and_check(LLAMA_INCLUDE_DIR "@PACKAGE_LLAMA_INCLUDE_INSTALL_DIR@")
-set_and_check(LLAMA_LIB_DIR "@PACKAGE_LLAMA_LIB_INSTALL_DIR@")
-set_and_check(LLAMA_BIN_DIR "@PACKAGE_LLAMA_BIN_INSTALL_DIR@")
+include(CMakeFindDependencyMacro)
+find_dependency(ggml CONFIG REQUIRED)
-find_package(ggml REQUIRED HINTS ${LLAMA_LIB_DIR}/cmake)
+include("${CMAKE_CURRENT_LIST_DIR}/llama-targets.cmake")
-find_library(llama_LIBRARY llama
- REQUIRED
- HINTS ${LLAMA_LIB_DIR}
- NO_CMAKE_FIND_ROOT_PATH
-)
-
-add_library(llama UNKNOWN IMPORTED)
-set_target_properties(llama
- PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
- INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;"
- IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
- IMPORTED_LOCATION "${llama_LIBRARY}"
- INTERFACE_COMPILE_FEATURES c_std_90
- POSITION_INDEPENDENT_CODE ON)
-
-check_required_components(Llama)
+check_required_components(llama)
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index bb168e8358aa..c72b54b27b0a 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -44,39 +44,50 @@ endif()
set(TARGET common)
+set(${TARGET}_HEADERS
+ arg.h
+ base64.hpp
+ chat-parser.h
+ chat.h
+ common.h
+ console.h
+ json-partial.h
+ json-schema-to-grammar.h
+ log.h
+ ngram-cache.h
+ regex-partial.h
+ sampling.h
+ speculative.h
+)
+
+list(TRANSFORM ${TARGET}_HEADERS PREPEND ${CMAKE_CURRENT_SOURCE_DIR}/)
+
add_library(${TARGET} STATIC
arg.cpp
- arg.h
- base64.hpp
chat-parser.cpp
chat-parser.h
chat-parser-xml-toolcall.h
chat-parser-xml-toolcall.cpp
chat.cpp
- chat.h
common.cpp
- common.h
console.cpp
console.h
download.cpp
download.h
http.h
json-partial.cpp
- json-partial.h
json-schema-to-grammar.cpp
llguidance.cpp
log.cpp
- log.h
ngram-cache.cpp
- ngram-cache.h
regex-partial.cpp
- regex-partial.h
sampling.cpp
- sampling.h
speculative.cpp
- speculative.h
+ ${${TARGET}_HEADERS}
)
+set_target_properties(${TARGET} PROPERTIES PUBLIC_HEADER "${${TARGET}_HEADERS}")
+
if (BUILD_SHARED_LIBS)
set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
endif()
@@ -143,7 +154,12 @@ if (LLAMA_LLGUIDANCE)
set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
endif ()
-target_include_directories(${TARGET} PUBLIC . ../vendor)
+target_include_directories(
+ ${TARGET}
+ PUBLIC
+ $${CMAKE_SOURCE_DIR}/vendor>
+ $)
+
target_compile_features (${TARGET} PUBLIC cxx_std_17)
target_link_libraries (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)
diff --git a/common/arg.cpp b/common/arg.cpp
index 45c0d1a72609..8c37edff6843 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1811,6 +1811,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.mmproj_use_gpu = false;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_NO_MMPROJ_OFFLOAD"));
+ add_opt(common_arg(
+ {"--mmproj-backend"}, "NAME",
+ "GPU backend for multimodal projector (e.g. CUDA, Metal, Vulkan)\n"
+ "if not specified, will use MTMD_BACKEND_DEVICE env var or default GPU backend",
+ [](common_params & params, const std::string & value) {
+ params.mmproj_backend = value;
+ }
+ ).set_examples({LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MAIN}));
add_opt(common_arg(
{"--image", "--audio"}, "FILE",
"path to an image or audio file. use with multimodal models, can be repeated if you have multiple files\n",
diff --git a/common/common.cpp b/common/common.cpp
index f07af1d86255..01b86c98a313 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -9,6 +9,8 @@
#include "log.h"
#include "llama.h"
#include "sampling.h"
+#include "chat.h"
+#include
#include
#include
@@ -1021,16 +1023,7 @@ static inline void common_init_sampler_from_model(
get_float(llama_model_meta_key_str(LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA), sparams.mirostat_eta, common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA);
}
-struct common_init_result common_init_from_params(common_params & params) {
- common_init_result iparams;
- auto mparams = common_model_params_to_llama(params);
-
- llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
- if (model == NULL) {
- LOG_ERR("%s: failed to load model '%s', try reducing --n-gpu-layers if you're running out of VRAM\n",
- __func__, params.model.path.c_str());
- return iparams;
- }
+struct common_init_result common_init_from_model_and_params(llama_model* model, common_init_result iparams, common_params & params) {
common_init_sampler_from_model(model, params.sampling);
@@ -1203,6 +1196,19 @@ struct common_init_result common_init_from_params(common_params & params) {
return iparams;
}
+struct common_init_result common_init_from_params(common_params & params) {
+ common_init_result iparams;
+ auto mparams = common_model_params_to_llama(params);
+
+ llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
+ if (model == NULL) {
+ LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());
+ return iparams;
+ }
+
+ return common_init_from_model_and_params(model, std::move(iparams), params);
+}
+
std::string get_model_endpoint() {
const char * model_endpoint_env = getenv("MODEL_ENDPOINT");
// We still respect the use of environment-variable "HF_ENDPOINT" for backward-compatibility.
@@ -1293,6 +1299,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
cparams.no_perf = params.no_perf;
cparams.op_offload = !params.no_op_offload;
cparams.swa_full = params.swa_full;
+ cparams.training = params.training;
cparams.kv_unified = params.kv_unified;
cparams.type_k = params.cache_type_k;
@@ -1738,3 +1745,333 @@ float lr_opt::get_lr(float epoch) const {
LOG_INF("epoch %.2g lr=%.2g\n", epoch, r);
return r;
}
+
+ggml_opt_dataset_t common_opt_sft_dataset_init(
+ struct llama_context * ctx,
+ const std::string & json_content,
+ int64_t stride,
+ const std::string & chat_template_path) {
+ using json = nlohmann::json;
+
+ const llama_vocab * vocab = llama_model_get_vocab(llama_get_model(ctx));
+ common_chat_templates_ptr chat_templates;
+ std::string chat_template_source;
+ if (!chat_template_path.empty()) {
+ std::ifstream tmpl_file(chat_template_path);
+ if (!tmpl_file.is_open()) {
+ LOG_ERR("Warning: Failed to open chat template file: %s\n", chat_template_path.c_str());
+ } else {
+ chat_template_source.assign(std::istreambuf_iterator(tmpl_file), std::istreambuf_iterator());
+ tmpl_file.close();
+ }
+ }
+
+ try {
+ chat_templates = common_chat_templates_init(llama_get_model(ctx), chat_template_source);
+ if (chat_template_source.empty()) {
+ LOG_INF("Using model's built-in chat template\n");
+ } else {
+ LOG_INF("Using custom chat template from: %s\n", chat_template_path.c_str());
+ }
+ } catch (const std::exception & e) {
+ if (!chat_template_path.empty()) {
+ LOG_ERR("Warning: Failed to parse chat template '%s': %s\n", chat_template_path.c_str(), e.what());
+ } else {
+ LOG_ERR("Warning: Failed to initialize chat template: %s\n", e.what());
+ }
+ }
+
+ std::vector conversations;
+ std::istringstream content_stream(json_content);
+
+ std::string line;
+ while (std::getline(content_stream, line)) {
+ if (line.empty() || line[0] == '#') continue;
+ try {
+ json conv = json::parse(line);
+ if (conv.contains("messages") && conv["messages"].is_array()) {
+ conversations.push_back(conv);
+ }
+ } catch (const json::exception & e) {
+ LOG_DBG("Warning: Failed to parse JSON line: %s\n", e.what());
+ }
+ }
+
+ if (conversations.empty()) {
+ LOG_ERR("Error: No valid conversations found\n");
+ return nullptr;
+ }
+ LOG_INF("Loaded %zu conversations\n", conversations.size());
+
+ const int64_t ne_datapoint = llama_n_ctx(ctx);
+ if (stride <= 0) stride = ne_datapoint;
+ if (stride > ne_datapoint) stride = ne_datapoint;
+
+ std::vector> all_tokenized_data;
+ std::vector> all_assistant_masks;
+
+ auto token_count_prefix = [&](const std::string & render, size_t char_count) -> size_t {
+ std::string prefix = render.substr(0, char_count);
+ auto t = common_tokenize(ctx, prefix, /*add_special=*/false, /*parse_special=*/true);
+ return t.size();
+ };
+
+ // TODO: The masking logic currently relies on string searching for specific role tags.
+ // While chat templates render appropriately, we must parse the rendered string to find
+ // the boundaries of the "assistant" role to calculate loss gracefully.
+ // Currently, this only supports ChatML (Qwen, etc.) and Gemma formats.
+ // A more reliable, model-agnostic method would require common_chat_templates_apply to
+ // return token role spans directly, which is not yet supported in common/chat.cpp.
+ const std::string START_TAG = "<|im_start|>";
+ const std::string START_SYS = "<|im_start|>system\n";
+ const std::string START_USR = "<|im_start|>user\n";
+ const std::string START_AST = "<|im_start|>assistant\n";
+ const std::string END_TAG = "<|im_end|>";
+ const std::string NL = "\n";
+
+ for (size_t i = 0; i < conversations.size(); ++i) {
+ const auto & messages = conversations[i]["messages"];
+ if (!messages.is_array() || messages.empty()) continue;
+
+ std::string render;
+
+ if (chat_templates) {
+ std::vector chat_msgs;
+ chat_msgs.reserve(messages.size());
+ for (const auto & msg : messages) {
+ if (!msg.contains("role") || !msg.contains("content")) {
+ continue;
+ }
+ common_chat_msg chat_msg;
+ chat_msg.role = msg["role"].get();
+ chat_msg.content = msg["content"].get();
+ chat_msgs.push_back(std::move(chat_msg));
+ }
+
+ if (!chat_msgs.empty()) {
+ common_chat_templates_inputs inputs;
+ inputs.messages = std::move(chat_msgs);
+ inputs.add_generation_prompt = false;
+ inputs.use_jinja = true;
+ try {
+ render = common_chat_templates_apply(chat_templates.get(), inputs).prompt;
+
+ size_t last_im_end = render.rfind("<|im_end|>");
+ if (last_im_end != std::string::npos) {
+ size_t end_pos = last_im_end + 10; // length of "<|im_end|>"
+ // Remove any trailing whitespace/newlines after the final <|im_end|>
+ while (end_pos < render.size() && (render[end_pos] == '\n' || render[end_pos] == '\r' || render[end_pos] == ' ')) {
+ end_pos++;
+ }
+ if (end_pos < render.size()) {
+ render = render.substr(0, last_im_end + 10); // Keep only up to
+ }
+ }
+ } catch (const std::exception & e) {
+ LOG_WRN("Warning: chat template rendering failed for conversation %zu: %s. Falling back to default ChatML rendering.\n",
+ i, e.what());
+ }
+ }
+ }
+
+ if (render.empty()) {
+ render.reserve(4096);
+ for (const auto & msg : messages) {
+ if (!msg.contains("role") || !msg.contains("content")) continue;
+ const std::string role = msg["role"].get();
+ const std::string content = msg["content"].get();
+
+ if (role == "system") {
+ render += START_SYS; render += content; render += END_TAG + NL;
+ } else if (role == "user") {
+ render += START_USR; render += content; render += END_TAG + NL;
+ } else if (role == "assistant") {
+ render += START_AST; render += content; render += END_TAG + NL;
+ }
+ }
+ }
+
+ if (render.empty()) {
+ continue;
+ }
+
+ struct Span { size_t lo, hi; };
+ std::vector assistant_spans;
+
+ {
+ bool is_gemma = render.find("model\n") != std::string::npos;
+
+ if (is_gemma) {
+ const std::string GEMMA_START = "model\n";
+ const std::string GEMMA_END = "";
+
+ size_t from = 0;
+ while (true) {
+ size_t open = render.find(GEMMA_START, from);
+ if (open == std::string::npos) break;
+ size_t lo = open;
+ size_t close = render.find(GEMMA_END, lo);
+ if (close == std::string::npos) {
+ assistant_spans.push_back({lo, render.size()});
+ break;
+ }
+
+ size_t hi = close + GEMMA_END.size();
+ if (hi < render.size() && render[hi] == '\n') {
+ hi++;
+ }
+ assistant_spans.push_back({lo, std::min(hi, render.size())});
+
+ from = hi;
+ }
+ } else {
+ size_t from = 0;
+ while (true) {
+ size_t open = render.find(START_AST, from);
+ if (open == std::string::npos) break;
+
+ // Include the role token ("assistant") and everything through the closing tag/newlines
+ size_t lo = open + START_TAG.size();
+ if (lo > render.size()) {
+ lo = render.size();
+ }
+
+ size_t close = render.find(END_TAG, open + START_AST.size());
+ if (close == std::string::npos) {
+ assistant_spans.push_back({lo, render.size()});
+ break;
+ }
+
+ size_t hi = close + END_TAG.size();
+ if (hi <= lo) {
+ lo = open;
+ hi = close + END_TAG.size();
+ }
+
+ assistant_spans.push_back({lo, std::min(hi, render.size())});
+
+ size_t next_from = hi;
+ from = next_from;
+ }
+ }
+ }
+
+ if (assistant_spans.empty()) {
+ LOG_WRN("Conversation %zu has no assistant spans\n", i);
+ continue;
+ }
+
+ auto tokens_full = common_tokenize(ctx, render, /*add_special=*/false, /*parse_special=*/true);
+ if (tokens_full.empty()) continue;
+
+ std::vector assistant_mask(tokens_full.size(), 0);
+ size_t assistant_token_count = 0;
+
+ for (const auto & sp : assistant_spans) {
+ size_t t_lo = token_count_prefix(render, sp.lo);
+ size_t t_hi = token_count_prefix(render, sp.hi);
+ if (t_lo > tokens_full.size()) t_lo = tokens_full.size();
+ if (t_hi > tokens_full.size()) t_hi = tokens_full.size();
+
+
+ for (size_t t = t_lo; t < t_hi; ++t) {
+ assistant_mask[t] = 1;
+ ++assistant_token_count;
+ }
+ }
+
+ if (assistant_token_count == 0) {
+ LOG_WRN("Warning: Conversation %zu has zero assistant tokens after masking\n", i);
+ continue;
+ }
+
+ all_tokenized_data.push_back(tokens_full);
+ all_assistant_masks.push_back(assistant_mask);
+ }
+
+ if (all_tokenized_data.empty()) {
+ LOG_ERR("ERROR: No valid training samples generated after processing %zu conversations\n", conversations.size());
+ return nullptr;
+ }
+
+ std::vector> final_samples;
+ std::vector> final_masks;
+
+ llama_token pad_token = llama_vocab_pad(vocab);
+ if (pad_token == LLAMA_TOKEN_NULL) {
+ pad_token = llama_vocab_eos(vocab);
+ }
+
+ for (size_t i = 0; i < all_tokenized_data.size(); ++i) {
+ const auto& conv_tokens = all_tokenized_data[i];
+ const auto& conv_mask = all_assistant_masks[i];
+
+ if ((int64_t)conv_tokens.size() > ne_datapoint) {
+ LOG_WRN("Skipping conversation %zu: too long (%zu tokens > %lld)\n", i, conv_tokens.size(), (long long)ne_datapoint);
+ continue;
+ }
+
+ size_t conv_assistant_tokens = 0;
+ for (int32_t mask_val : conv_mask) {
+ if (mask_val == 1) conv_assistant_tokens++;
+ }
+
+ if (conv_assistant_tokens == 0) {
+ LOG_WRN("Skipping conversation %zu: no assistant tokens\n", i);
+ continue;
+ }
+
+ std::vector sample_tokens = conv_tokens;
+ std::vector sample_mask = conv_mask;
+
+ sample_tokens.resize(ne_datapoint, pad_token);
+ sample_mask.resize(ne_datapoint, 0); // Padding tokens are not trained on
+
+ final_samples.push_back(sample_tokens);
+ final_masks.push_back(sample_mask);
+ }
+
+ all_tokenized_data = std::move(final_samples);
+ all_assistant_masks = std::move(final_masks);
+
+ const int64_t ndata = all_tokenized_data.size();
+
+ ggml_opt_dataset_t result = ggml_opt_dataset_init_with_masks(
+ GGML_TYPE_I32, GGML_TYPE_I32, GGML_TYPE_I32,
+ /*ne_datapoint=*/ne_datapoint, /*ne_label=*/ne_datapoint, /*ne_mask=*/ne_datapoint,
+ /*ndata=*/ndata, /*ndata_shard=*/1);
+
+ if (result == nullptr) {
+ return nullptr;
+ }
+
+ int32_t * data = (int32_t *) ggml_opt_dataset_data(result)->data;
+ int32_t * labels = (int32_t *) ggml_opt_dataset_labels(result)->data;
+ int32_t * masks = (int32_t *) ggml_opt_dataset_masks(result)->data;
+
+ for (int64_t idata = 0; idata < ndata; ++idata) {
+ const auto & sample_tokens = all_tokenized_data[idata];
+ const auto & sample_mask = all_assistant_masks[idata];
+
+ // inputs
+ for (int64_t i = 0; i < ne_datapoint; ++i) {
+ data[idata * ne_datapoint + i] = sample_tokens[i];
+ }
+
+ // labels: Set actual next tokens for ALL positions (masked cross-entropy needs real tokens)
+ for (int64_t i = 0; i < ne_datapoint - 1; ++i) {
+ // Always set the actual next token - masking is handled separately
+ labels[idata * ne_datapoint + i] = sample_tokens[i + 1];
+ }
+ labels[idata * ne_datapoint + (ne_datapoint - 1)] = sample_tokens[ne_datapoint - 1]; // last token predicts itself (will be masked)
+
+ // masks: indicate which preds should be trained on (shifted by 1 from sample_mask)
+ // Since we predict token[i+1] from token[i], we train when token[i+1] is assistant
+ for (int64_t i = 0; i < ne_datapoint - 1; ++i) {
+ masks[idata * ne_datapoint + i] = (i + 1 < ne_datapoint && sample_mask[i + 1] == 1) ? 1 : 0;
+ }
+ masks[idata * ne_datapoint + (ne_datapoint - 1)] = 0;
+ }
+
+ return result;
+}
diff --git a/common/common.h b/common/common.h
index 6e6b2c1cab69..1fc687eab8e6 100644
--- a/common/common.h
+++ b/common/common.h
@@ -415,6 +415,7 @@ struct common_params {
bool warmup = true; // warmup run
bool check_tensors = false; // validate tensor data
bool no_op_offload = false; // globally disable offload host tensor operations to device
+ bool training = false; // enable training mode (affects LoRA K/V gradient flow)
bool no_extra_bufts = false; // disable extra buffer types (used for weight repacking)
bool no_host = false; // bypass host buffer allowing extra buffers to be used
@@ -428,6 +429,7 @@ struct common_params {
// multimodal models (see tools/mtmd)
struct common_params_model mmproj;
bool mmproj_use_gpu = true; // use GPU for multimodal model
+ std::string mmproj_backend = ""; // GPU backend for multimodal model (e.g. "CUDA", "Metal", "Vulkan")
bool no_mmproj = false; // explicitly disable multimodal model
std::vector image; // path to image file(s)
int image_min_tokens = -1;
@@ -664,6 +666,8 @@ struct common_init_result {
};
struct common_init_result common_init_from_params(common_params & params);
+struct common_init_result common_init_from_model_and_params(llama_model * model, common_init_result iparams,
+ common_params & params);
struct llama_model_params common_model_params_to_llama ( common_params & params);
struct llama_context_params common_context_params_to_llama(const common_params & params);
@@ -804,3 +808,8 @@ ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std
// "adamw" or "sgd" (case insensitive)
enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);
+ggml_opt_dataset_t common_opt_sft_dataset_init(
+ struct llama_context * ctx,
+ const std::string & json_content,
+ int64_t stride,
+ const std::string & chat_template_path = "");
diff --git a/common/log.cpp b/common/log.cpp
index b6c9ff79a43d..ef244bddd321 100644
--- a/common/log.cpp
+++ b/common/log.cpp
@@ -92,7 +92,14 @@ struct common_log_entry {
// signals the worker thread to stop
bool is_end;
- void print(FILE * file = nullptr) const {
+ void print(FILE * file = nullptr, ggml_log_callback callback = nullptr, void * callback_user_data = nullptr) const {
+ // if callback is provided, use it instead of printing
+ if (callback != nullptr) {
+ callback(level, msg.data(), callback_user_data);
+ return;
+ }
+
+
FILE * fcur = file;
if (!fcur) {
// stderr displays DBG messages only when their verbosity level is not higher than the threshold
@@ -150,6 +157,8 @@ struct common_log {
timestamps = false;
running = false;
t_start = t_us();
+ callback = nullptr;
+ callback_user_data = nullptr;
// initial message size - will be expanded if longer messages arrive
entries.resize(capacity);
@@ -191,6 +200,10 @@ struct common_log {
// worker thread copies into this
common_log_entry cur;
+ // custom callback for log messages
+ ggml_log_callback callback;
+ void * callback_user_data;
+
public:
void add(enum ggml_log_level level, const char * fmt, va_list args) {
std::lock_guard lock(mtx);
@@ -281,11 +294,15 @@ struct common_log {
thrd = std::thread([this]() {
while (true) {
+ ggml_log_callback cb = nullptr;
+ void * cb_user_data = nullptr;
{
std::unique_lock lock(mtx);
cv.wait(lock, [this]() { return head != tail; });
cur = entries[head];
+ cb = callback;
+ cb_user_data = callback_user_data;
head = (head + 1) % entries.size();
}
@@ -294,7 +311,7 @@ struct common_log {
break;
}
- cur.print(); // stdout and stderr
+ cur.print(nullptr, cb, cb_user_data); // stdout and stderr or callback
if (file) {
cur.print(file);
@@ -376,6 +393,15 @@ struct common_log {
this->timestamps = timestamps;
}
+
+ void set_callback(ggml_log_callback cb, void * user_data) {
+ pause();
+
+ this->callback = cb;
+ this->callback_user_data = user_data;
+
+ resume();
+ }
};
//
@@ -462,3 +488,7 @@ void common_log_default_callback(enum ggml_log_level level, const char * text, v
common_log_add(common_log_main(), level, "%s", text);
}
}
+
+void common_log_set_callback(struct common_log * log, ggml_log_callback callback, void * user_data) {
+ log->set_callback(callback, user_data);
+}
diff --git a/common/log.h b/common/log.h
index b24f5f000a6e..8b338d24ae18 100644
--- a/common/log.h
+++ b/common/log.h
@@ -85,6 +85,11 @@ void common_log_set_colors (struct common_log * log, log_colors colors); // n
void common_log_set_prefix (struct common_log * log, bool prefix); // whether to output prefix to each log
void common_log_set_timestamps(struct common_log * log, bool timestamps); // whether to output timestamps in the prefix
+// set a custom callback to handle log messages instead of printing to stdout/stderr
+// if callback is NULL, reverts to default printing behavior
+// note: the callback will be called from the worker thread
+void common_log_set_callback(struct common_log * log, ggml_log_callback callback, void * user_data); // not thread-safe
+
// helper macros for logging
// use these to avoid computing log arguments if the verbosity of the log is higher than the threshold
//
diff --git a/common_test/CMakeLists.txt b/common_test/CMakeLists.txt
new file mode 100644
index 000000000000..44903612e534
--- /dev/null
+++ b/common_test/CMakeLists.txt
@@ -0,0 +1,15 @@
+# common_test library for load_into_memory.h and uint8-buff-stream.h
+
+set(TARGET llama-common-test)
+
+add_library(${TARGET} INTERFACE)
+
+target_include_directories(${TARGET} INTERFACE
+ ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+target_compile_definitions(${TARGET} INTERFACE LLAMA_COMMON_TEST_HEADERS)
+
+target_compile_features(${TARGET} INTERFACE cxx_std_17)
+
+target_link_libraries(${TARGET} INTERFACE common)
diff --git a/common_test/load_into_memory.h b/common_test/load_into_memory.h
new file mode 100644
index 000000000000..1c84e84cce82
--- /dev/null
+++ b/common_test/load_into_memory.h
@@ -0,0 +1,220 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+// header-only utilities to showcase how to directly load a model from memory
+#include "uint8-buff-stream-wrapper.h"
+
+namespace {
+bool is_split_file(const char * const model_path) {
+ if (!model_path) {
+ fprintf(stderr, "No model file provided\n");
+ exit(EXIT_FAILURE);
+ }
+
+ std::string path(model_path);
+ return path.find("-of-") != std::string::npos;
+}
+
+std::vector load_file_into_buffer(const char * const model_path) {
+ std::ifstream file_stream(model_path, std::ios::binary | std::ios::ate);
+ if (!file_stream) {
+ fprintf(stderr, "Failed to open file %s for reading into streambuf\n", model_path);
+ exit(EXIT_FAILURE);
+ }
+
+ const size_t file_size = file_stream.tellg();
+ file_stream.seekg(0, std::ios::beg);
+
+ static_assert(sizeof(std::uint8_t) == sizeof(char), "uint8_t must be same size as char");
+ std::vector buffer(file_size);
+ if (!file_stream.read((char *) buffer.data(), file_size)) {
+ fprintf(stderr, "Failed to read entire file into buffer\n");
+ exit(EXIT_FAILURE);
+ }
+
+ return buffer;
+}
+
+std::unique_ptr> load_file_into_streambuf(const char * const model_path) {
+ return std::make_unique(load_file_into_buffer(model_path));
+}
+
+struct file_entry {
+ std::string path;
+ std::unique_ptr> streambuf;
+};
+
+std::vector load_files_into_streambuf(const char * const model_path) {
+ std::vector files;
+
+ // Extract pattern from first file path
+ std::string path(model_path);
+
+ // Split by '-'
+ std::vector parts;
+ std::stringstream ss(path);
+ std::string item;
+ while (std::getline(ss, item, '-')) {
+ parts.push_back(item);
+ }
+
+ // Split the last part by '.'
+ std::string last_part = parts.back();
+ parts.pop_back();
+ size_t dot_pos = last_part.find('.');
+ if (dot_pos != std::string::npos) {
+ parts.push_back(last_part.substr(0, dot_pos));
+ parts.push_back(last_part.substr(dot_pos + 1)); // extension
+ } else {
+ parts.push_back(last_part);
+ }
+
+ // Check if we have enough parts
+ if (parts.size() < 4) {
+ fprintf(stderr, "Model path does not contain expected pattern\n");
+ exit(EXIT_FAILURE);
+ }
+
+ // Get total files from [-2] position (before the extension)
+ int total_files = std::stoi(parts[parts.size() - 2]);
+
+ // Get base path by joining all parts except -start-of-end.gguf
+ std::string base_path;
+ for (size_t i = 0; i < parts.size() - 4; i++) {
+ if (i > 0) {
+ base_path += "-";
+ }
+ base_path += parts[i];
+ }
+
+ for (int i = 1; i <= total_files; i++) {
+ char numbered_path[1024];
+ snprintf(numbered_path, sizeof(numbered_path), "%s-%05d-of-%05d.gguf", base_path.c_str(), i, total_files);
+
+ files.push_back({ numbered_path, load_file_into_streambuf(numbered_path) });
+ }
+
+ return files;
+}
+
+file_entry load_tensor_list_file(const char * const model_path) {
+ std::string path(model_path);
+
+ // Split by '-'
+ std::vector parts;
+ std::stringstream ss(path);
+ std::string item;
+ while (std::getline(ss, item, '-')) {
+ parts.push_back(item);
+ }
+
+ // Split the last part by '.'
+ std::string last_part = parts.back();
+ parts.pop_back();
+ size_t dot_pos = last_part.find('.');
+ if (dot_pos != std::string::npos) {
+ parts.push_back(last_part.substr(0, dot_pos));
+ parts.push_back(last_part.substr(dot_pos + 1)); // extension
+ } else {
+ parts.push_back(last_part);
+ }
+
+ // Check if we have enough parts
+ if (parts.size() < 4) {
+ fprintf(stderr, "Model path does not contain expected pattern\n");
+ exit(EXIT_FAILURE);
+ }
+
+ // Get base path by joining all parts except -start-of-end.gguf
+ std::string base_path;
+ for (size_t i = 0; i < parts.size() - 4; i++) {
+ if (i > 0) {
+ base_path += "-";
+ }
+ base_path += parts[i];
+ }
+
+ // Construct tensor list file path
+ std::string tensor_list_path = base_path + ".tensors.txt";
+
+ printf("Loading tensor list file: %s\n", tensor_list_path.c_str());
+ return { tensor_list_path, load_file_into_streambuf(tensor_list_path.c_str()) };
+}
+
+llama_model * load_model_from_memory_configuration(const char * model_path, llama_model_params & model_params) {
+ llama_model * model;
+ std::chrono::steady_clock::time_point load_start_time;
+ if (getenv("LLAMA_EXAMPLE_MEMORY_BUFFER")) {
+ std::vector buffer = load_file_into_buffer(model_path);
+ fprintf(stdout, "%s: loading model from memory buffer\n", __func__);
+ load_start_time = std::chrono::steady_clock::now();
+ model = llama_model_load_from_buffer(std::move(buffer), model_params);
+ } else if (getenv("LLAMA_EXAMPLE_MEMORY_BUFFER_SPLIT")) {
+ file_entry tensor_list_file = load_tensor_list_file(model_path);
+ std::vector files = load_files_into_streambuf(model_path);
+ fprintf(stdout, "%s: loading model from %zu file streambufs\n", __func__, files.size());
+
+ std::vector file_paths;
+ for (const auto & file : files) {
+ printf("Found file %s with streambuf\n", file.path.c_str());
+ file_paths.push_back(file.path.c_str());
+ }
+
+ load_start_time = std::chrono::steady_clock::now();
+ const char * async_load_context = "test-model-load";
+ std::thread fulfill_thread([&files, &tensor_list_file, &async_load_context]() {
+ const bool success = llama_model_load_fulfill_split_future(
+ tensor_list_file.path.c_str(), async_load_context, std::move(tensor_list_file.streambuf));
+ printf("Fulfilling tensor list file %s: %s\n", tensor_list_file.path.c_str(),
+ success ? "success" : "failure");
+ if (!success) {
+ exit(EXIT_FAILURE);
+ }
+
+ for (auto & file : files) {
+ const bool success = llama_model_load_fulfill_split_future(file.path.c_str(), async_load_context,
+ std::move(file.streambuf));
+ printf("Fulfilling file %s with streambuf: %s\n", file.path.c_str(), success ? "success" : "failure");
+ if (!success) {
+ exit(EXIT_FAILURE);
+ }
+ }
+ });
+ fprintf(stderr, "Loading model from splits\n");
+ model = llama_model_load_from_split_futures(file_paths.data(), file_paths.size(), async_load_context,
+ tensor_list_file.path.c_str(), model_params);
+ fulfill_thread.join();
+ } else if (getenv("LLAMA_EXAMPLE_FROM_FILE")) {
+ load_start_time = std::chrono::steady_clock::now();
+ model = llama_model_load_from_file(model_path, model_params);
+ } else {
+ return nullptr;
+ }
+
+ if (model == NULL) {
+ fprintf(stderr, "%s: error: unable to load model\n", __func__);
+ exit(1);
+ }
+ std::chrono::steady_clock::time_point load_end_time = std::chrono::steady_clock::now();
+ std::chrono::duration load_duration = load_end_time - load_start_time;
+ fprintf(stdout, "%s: loading model took %f seconds\n", __func__, load_duration.count());
+ return model;
+}
+
+bool memory_configuration_env_is_set() {
+ return getenv("LLAMA_EXAMPLE_MEMORY_BUFFER") || getenv("LLAMA_EXAMPLE_MEMORY_BUFFER_SPLIT") ||
+ getenv("LLAMA_EXAMPLE_FROM_FILE");
+}
+} // namespace
diff --git a/common_test/uint8-buff-stream-wrapper.h b/common_test/uint8-buff-stream-wrapper.h
new file mode 100644
index 000000000000..3a03721b98c0
--- /dev/null
+++ b/common_test/uint8-buff-stream-wrapper.h
@@ -0,0 +1,5 @@
+#pragma once
+
+// Wrapper to include the specific header from src
+#include "uint8-buff-stream.h"
+
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 8ddb6d04cd99..307866857c8a 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -3029,18 +3029,47 @@ def prepare_tensors(self):
super().prepare_tensors()
-@ModelBase.register("BitnetForCausalLM")
+@ModelBase.register("BitnetForCausalLM", "BitNetForCausalLM")
class BitnetModel(TextModel):
model_arch = gguf.MODEL_ARCH.BITNET
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._bitnet_weight_scales: dict[str, torch.Tensor] = {}
+
def set_vocab(self):
- self._set_vocab_sentencepiece()
+ if (self.dir_model / "tokenizer.model").is_file():
+ self._set_vocab_sentencepiece()
+ else:
+ self._set_vocab_gpt2()
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
self.gguf_writer.add_rope_scaling_factor(1.0)
+ @staticmethod
+ def _unpack_bitnet_weights(packed: torch.Tensor) -> torch.Tensor:
+ if packed.dtype != torch.uint8:
+ raise ValueError(f"Expected packed BitNet weights to be torch.uint8, got {packed.dtype}")
+
+ values_per_item = 4
+ rows = packed.shape[0]
+ rest = packed.shape[1:]
+
+ unpacked_chunks: list[torch.Tensor] = []
+ mapping = torch.tensor([-1.0, 0.0, 1.0, 0.0], dtype=torch.float32, device=packed.device)
+
+ for i in range(values_per_item):
+ chunk = (packed >> (2 * i)) & 0x03
+ chunk = mapping[chunk.long()].reshape((rows, *rest))
+ unpacked_chunks.append(chunk)
+
+ if not unpacked_chunks:
+ raise ValueError("Failed to unpack BitNet weights: no chunks produced")
+
+ return torch.cat(unpacked_chunks, dim=0)
+
def weight_quant(self, weight: Tensor) -> Tensor:
dtype = weight.dtype
weight = weight.float()
@@ -3053,8 +3082,37 @@ def weight_quant(self, weight: Tensor) -> Tensor:
return result.type(dtype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ weight_scale_suffix = ".weight_scale"
+ if name.endswith(weight_scale_suffix):
+ weight_name = name[:-len(weight_scale_suffix)] + ".weight"
+ mapped_weight_name = self.map_tensor_name(weight_name)
+ if isinstance(data_torch, LazyTorchTensor):
+ data_torch = LazyTorchTensor.to_eager(data_torch)
+
+ scale_tensor = data_torch.to(torch.float32)
+ self._bitnet_weight_scales[mapped_weight_name] = scale_tensor
+ return []
+
new_name = self.map_tensor_name(name)
+ ternary_weight = False
+
+ if name.endswith(".weight"):
+ scale_tensor = self._bitnet_weight_scales.pop(new_name, None)
+ if scale_tensor is not None:
+ scale_tensor = scale_tensor.to(torch.float32)
+ if scale_tensor.numel() != 1:
+ raise ValueError(f"Expected scalar weight_scale for '{name}', got shape {tuple(scale_tensor.shape)}")
+
+ if isinstance(data_torch, LazyTorchTensor):
+ data_torch = LazyTorchTensor.to_eager(data_torch)
+
+ packed = data_torch.to(torch.uint8)
+ unpacked = self._unpack_bitnet_weights(packed)
+ scale_value = scale_tensor.reshape(-1)[0].item()
+ data_torch = unpacked * scale_value
+ ternary_weight = True
+
if any(self.match_model_tensor_name(new_name, key, bid) for key in [
gguf.MODEL_TENSOR.ATTN_Q,
gguf.MODEL_TENSOR.ATTN_K,
@@ -3063,7 +3121,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
gguf.MODEL_TENSOR.FFN_UP,
gguf.MODEL_TENSOR.FFN_DOWN,
gguf.MODEL_TENSOR.FFN_GATE,
- ]):
+ ]) and not ternary_weight:
# transform weight into 1/0/-1 (in fp32)
data_torch = self.weight_quant(data_torch)
diff --git a/docs/backend/hexagon/CMakeUserPresets.json b/docs/backend/hexagon/CMakeUserPresets.json
index e0b19db0f5a2..8828ea4aa88a 100644
--- a/docs/backend/hexagon/CMakeUserPresets.json
+++ b/docs/backend/hexagon/CMakeUserPresets.json
@@ -1,4 +1,4 @@
-{
+{
"version": 4,
"configurePresets": [
{
diff --git a/examples/embedding/CMakeLists.txt b/examples/embedding/CMakeLists.txt
index 809040307d2c..be3d7b17e157 100644
--- a/examples/embedding/CMakeLists.txt
+++ b/examples/embedding/CMakeLists.txt
@@ -1,5 +1,5 @@
set(TARGET llama-embedding)
add_executable(${TARGET} embedding.cpp)
install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common llama llama-common-test ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_17)
diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp
index fe91b308cdc0..8f7899aa2f5b 100644
--- a/examples/embedding/embedding.cpp
+++ b/examples/embedding/embedding.cpp
@@ -1,15 +1,25 @@
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
#include "arg.h"
#include "common.h"
+#include "llama-cpp.h"
#include "log.h"
-#include "llama.h"
-
-#include
-#include
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
+#ifdef LLAMA_COMMON_TEST_HEADERS
+#include "load_into_memory.h"
+#endif
+
static std::vector split_lines(const std::string & s, const std::string & separator = "\n") {
std::vector lines;
size_t start = 0;
@@ -131,7 +141,20 @@ int main(int argc, char ** argv) {
llama_numa_init(params.numa);
// load the model
- common_init_result llama_init = common_init_from_params(params);
+ common_init_result llama_init;
+
+#ifdef LLAMA_COMMON_TEST_HEADERS
+ if (memory_configuration_env_is_set()) {
+ llama_model_params mparams = common_model_params_to_llama(params);
+ common_init_result iparams;
+ llama_model * model = load_model_from_memory_configuration(params.model.path.c_str(), mparams);
+ llama_init = common_init_from_model_and_params(model, std::move(iparams), params);
+ } else {
+ llama_init = common_init_from_params(params);
+ }
+#else
+ llama_init = common_init_from_params(params);
+#endif
llama_model * model = llama_init.model.get();
llama_context * ctx = llama_init.context.get();
diff --git a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt
index 6119fe09b0cb..572eb9436dc3 100644
--- a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt
+++ b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt
@@ -21,6 +21,12 @@ project("llama-android")
# Also provides "common"
#FetchContent_MakeAvailable(llama)
+set(LLAMA_ROOT_DIR "../../../../../..")
+
+# Set global include directories that will be used by all targets
+include_directories(${LLAMA_ROOT_DIR}/include)
+include_directories(${LLAMA_ROOT_DIR}/vendor)
+
# Creates and names a library, sets it as either STATIC
# or SHARED, and provides the relative paths to its source code.
# You can define multiple libraries, and CMake builds them for you.
@@ -32,7 +38,7 @@ project("llama-android")
#
#load local llama.cpp
-add_subdirectory(../../../../../../ build-llama)
+add_subdirectory(${LLAMA_ROOT_DIR}/ build-llama)
# In order to load a library into your app from Java/Kotlin, you must call
# System.loadLibrary() and pass the name of the library defined here;
@@ -51,3 +57,9 @@ target_link_libraries(${CMAKE_PROJECT_NAME}
common
android
log)
+
+# Add include directories for llama.h and common.h
+target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE
+ ${LLAMA_ROOT_DIR}/include
+ ${LLAMA_ROOT_DIR}/src
+ ${LLAMA_ROOT_DIR}/common)
diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
index dc2bafc88b17..7d78eaaa10f2 100644
--- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
+++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
@@ -5,6 +5,16 @@ enum LlamaError: Error {
case couldNotInitializeContext
}
+struct LlamaRuntimeOptions {
+ var contextLength: Int32
+ var nGpuLayers: Int32
+ var seed: UInt32
+ var temperature: Float
+ var topP: Float
+ var topK: Int32
+ var flashAttention: Bool
+}
+
func llama_batch_clear(_ batch: inout llama_batch) {
batch.n_tokens = 0
}
@@ -25,9 +35,10 @@ actor LlamaContext {
private var model: OpaquePointer
private var context: OpaquePointer
private var vocab: OpaquePointer
- private var sampling: UnsafeMutablePointer
+ private var sampling: UnsafeMutablePointer?
private var batch: llama_batch
private var tokens_list: [llama_token]
+ private var runtimeOptions: LlamaRuntimeOptions
var is_done: Bool = false
/// This variable is used to store temporarily invalid cchars
@@ -38,34 +49,80 @@ actor LlamaContext {
var n_decode: Int32 = 0
- init(model: OpaquePointer, context: OpaquePointer) {
+ init(model: OpaquePointer, context: OpaquePointer, options: LlamaRuntimeOptions) {
self.model = model
self.context = context
self.tokens_list = []
- self.batch = llama_batch_init(512, 0, 1)
+ self.batch = llama_batch_init(max(Int32(512), options.contextLength), 0, 1)
self.temporary_invalid_cchars = []
- let sparams = llama_sampler_chain_default_params()
- self.sampling = llama_sampler_chain_init(sparams)
- llama_sampler_chain_add(self.sampling, llama_sampler_init_temp(0.4))
- llama_sampler_chain_add(self.sampling, llama_sampler_init_dist(1234))
+ self.runtimeOptions = options
+ self.n_len = options.contextLength
vocab = llama_model_get_vocab(model)
+
+ let chainParams = llama_sampler_chain_default_params()
+ let initialChain = llama_sampler_chain_init(chainParams)
+
+ if options.topK > 0 {
+ llama_sampler_chain_add(initialChain, llama_sampler_init_top_k(options.topK))
+ }
+
+ let clampedTopP = max(0.0, min(Double(options.topP), 1.0))
+ llama_sampler_chain_add(initialChain, llama_sampler_init_top_p(Float(clampedTopP), 1))
+
+ let clampedTemp = max(0.0, Double(options.temperature))
+ llama_sampler_chain_add(initialChain, llama_sampler_init_temp(Float(clampedTemp)))
+
+ let seed = options.seed == 0 ? UInt32.max : options.seed
+ llama_sampler_chain_add(initialChain, llama_sampler_init_dist(seed))
+
+ sampling = initialChain
}
deinit {
- llama_sampler_free(sampling)
+ if let sampling {
+ llama_sampler_free(sampling)
+ }
llama_batch_free(batch)
llama_model_free(model)
llama_free(context)
llama_backend_free()
}
- static func create_context(path: String) throws -> LlamaContext {
+ private func rebuildSamplerChain() {
+ let chainParams = llama_sampler_chain_default_params()
+ let newChain = llama_sampler_chain_init(chainParams)
+
+ if runtimeOptions.topK > 0 {
+ llama_sampler_chain_add(newChain, llama_sampler_init_top_k(runtimeOptions.topK))
+ }
+
+ let clampedTopP = max(0.0, min(runtimeOptions.topP, 1.0))
+ llama_sampler_chain_add(newChain, llama_sampler_init_top_p(clampedTopP, 1))
+
+ let clampedTemp = max(0.0, Double(runtimeOptions.temperature))
+ llama_sampler_chain_add(newChain, llama_sampler_init_temp(Float(clampedTemp)))
+
+ let seed = runtimeOptions.seed == 0 ? UInt32.max : runtimeOptions.seed
+ llama_sampler_chain_add(newChain, llama_sampler_init_dist(seed))
+
+ if let sampling {
+ llama_sampler_free(sampling)
+ }
+
+ sampling = newChain
+ }
+
+ static func create_context(path: String, options: LlamaRuntimeOptions) throws -> LlamaContext {
llama_backend_init()
var model_params = llama_model_default_params()
#if targetEnvironment(simulator)
model_params.n_gpu_layers = 0
print("Running on simulator, force use n_gpu_layers = 0")
+#else
+ if options.nGpuLayers >= 0 {
+ model_params.n_gpu_layers = options.nGpuLayers
+ }
#endif
let model = llama_model_load_from_file(path, model_params)
guard let model else {
@@ -77,9 +134,10 @@ actor LlamaContext {
print("Using \(n_threads) threads")
var ctx_params = llama_context_default_params()
- ctx_params.n_ctx = 2048
+ ctx_params.n_ctx = UInt32(options.contextLength)
ctx_params.n_threads = Int32(n_threads)
ctx_params.n_threads_batch = Int32(n_threads)
+ ctx_params.flash_attn_type = options.flashAttention ? LLAMA_FLASH_ATTN_TYPE_ENABLED : LLAMA_FLASH_ATTN_TYPE_DISABLED
let context = llama_init_from_model(model, ctx_params)
guard let context else {
@@ -87,7 +145,13 @@ actor LlamaContext {
throw LlamaError.couldNotInitializeContext
}
- return LlamaContext(model: model, context: context)
+ return LlamaContext(model: model, context: context, options: options)
+ }
+
+ func updateSampler(options: LlamaRuntimeOptions) {
+ runtimeOptions = options
+ n_len = options.contextLength
+ rebuildSamplerChain()
}
func model_info() -> String {
@@ -151,6 +215,10 @@ actor LlamaContext {
func completion_loop() -> String {
var new_token_id: llama_token = 0
+ guard let sampling else {
+ return ""
+ }
+
new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1)
if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len {
diff --git a/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj b/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj
index 6f08fe220a9d..401a9371f293 100644
--- a/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj
+++ b/examples/llama.swiftui/llama.swiftui.xcodeproj/project.pbxproj
@@ -20,6 +20,7 @@
DD84C9FD2D747FED007778EC /* llama.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = DD84C9FC2D747FED007778EC /* llama.xcframework */; };
DD84C9FE2D747FED007778EC /* llama.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = DD84C9FC2D747FED007778EC /* llama.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
F1FE20E22B465ECA00B45541 /* LoadCustomButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = F1FE20E12B465EC900B45541 /* LoadCustomButton.swift */; };
+ D1A2F0012E0C8E8B00A1B1C0 /* FinetuneBridge.mm in Sources */ = {isa = PBXBuildFile; fileRef = D1A2F0002E0C8E8B00A1B1C0 /* FinetuneBridge.mm */; };
/* End PBXBuildFile section */
/* Begin PBXCopyFilesBuildPhase section */
@@ -51,6 +52,9 @@
DD84C9FC2D747FED007778EC /* llama.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = llama.xcframework; path = "../../build-apple/llama.xcframework"; sourceTree = ""; };
DF2D2FE72B4A59BE00FCB72D /* llama.cpp */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = llama.cpp; path = ../..; sourceTree = ""; };
F1FE20E12B465EC900B45541 /* LoadCustomButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LoadCustomButton.swift; sourceTree = ""; };
+ D1A2F0002E0C8E8B00A1B1C0 /* FinetuneBridge.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FinetuneBridge.mm; sourceTree = ""; };
+ D1A2F0022E0C8E8B00A1B1C0 /* FinetuneBridge.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FinetuneBridge.h; sourceTree = ""; };
+ D1A2F0032E0C8E8B00A1B1C0 /* llama_swiftui-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "llama_swiftui-Bridging-Header.h"; sourceTree = ""; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@@ -93,6 +97,7 @@
8A3F84102AC4BD85005E2EE8 /* Resources */,
8A9F7C4B2AC332DC008AE1EA /* Models */,
8A9F7C4A2AC332BF008AE1EA /* UI */,
+ D1A2F0062E0C8E8B00A1B1C0 /* Bridging */,
8A1C83762AC328BD0096AF73 /* llama_swiftuiApp.swift */,
8A1C837A2AC328BE0096AF73 /* Assets.xcassets */,
);
@@ -109,19 +114,19 @@
name = Frameworks;
sourceTree = "";
};
- 8A3F84102AC4BD85005E2EE8 /* Resources */ = {
+ 8A3F84112AC4BD8C005E2EE8 /* models */ = {
isa = PBXGroup;
children = (
- 8A3F84112AC4BD8C005E2EE8 /* models */,
);
- path = Resources;
+ path = models;
sourceTree = "";
};
- 8A3F84112AC4BD8C005E2EE8 /* models */ = {
+ 8A3F84102AC4BD85005E2EE8 /* Resources */ = {
isa = PBXGroup;
children = (
+ 8A3F84112AC4BD8C005E2EE8 /* models */,
);
- path = models;
+ path = Resources;
sourceTree = "";
};
8A907F312AC7134E006146EA /* llama.cpp.swift */ = {
@@ -132,6 +137,16 @@
path = llama.cpp.swift;
sourceTree = "";
};
+ D1A2F0062E0C8E8B00A1B1C0 /* Bridging */ = {
+ isa = PBXGroup;
+ children = (
+ D1A2F0022E0C8E8B00A1B1C0 /* FinetuneBridge.h */,
+ D1A2F0032E0C8E8B00A1B1C0 /* llama_swiftui-Bridging-Header.h */,
+ D1A2F0002E0C8E8B00A1B1C0 /* FinetuneBridge.mm */,
+ );
+ path = Bridging;
+ sourceTree = "";
+ };
8A9F7C4A2AC332BF008AE1EA /* UI */ = {
isa = PBXGroup;
children = (
@@ -230,6 +245,7 @@
F1FE20E22B465ECA00B45541 /* LoadCustomButton.swift in Sources */,
8A907F332AC7138A006146EA /* LibLlama.swift in Sources */,
8A9F7C4D2AC332EE008AE1EA /* LlamaState.swift in Sources */,
+ D1A2F0012E0C8E8B00A1B1C0 /* FinetuneBridge.mm in Sources */,
8A1C83792AC328BD0096AF73 /* ContentView.swift in Sources */,
8A1C83772AC328BD0096AF73 /* llama_swiftuiApp.swift in Sources */,
7FA3D2B32B2EA2F600543F92 /* DownloadButton.swift in Sources */,
@@ -364,9 +380,10 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
+ CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
- DEVELOPMENT_TEAM = K5UQJPP73A;
+ DEVELOPMENT_TEAM = 3LT8Z8ZRCG;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
@@ -380,14 +397,16 @@
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.0;
- PRODUCT_BUNDLE_IDENTIFIER = "com.bachittle.llama-swift";
+ PRODUCT_BUNDLE_IDENTIFIER = "llama-collabora";
PRODUCT_NAME = "$(TARGET_NAME)";
+ PROVISIONING_PROFILE_SPECIFIER = "";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator xros xrsimulator";
SUPPORTS_XR_DESIGNED_FOR_IPHONE_IPAD = NO;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2,7";
+ SWIFT_OBJC_BRIDGING_HEADER = "llama.swiftui/Bridging/llama_swiftui-Bridging-Header.h";
};
name = Debug;
};
@@ -396,9 +415,10 @@
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
+ CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
- DEVELOPMENT_TEAM = K5UQJPP73A;
+ DEVELOPMENT_TEAM = 3LT8Z8ZRCG;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
@@ -412,13 +432,15 @@
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.0;
- PRODUCT_BUNDLE_IDENTIFIER = "com.bachittle.llama-swift";
+ PRODUCT_BUNDLE_IDENTIFIER = "llama-collabora";
PRODUCT_NAME = "$(TARGET_NAME)";
+ PROVISIONING_PROFILE_SPECIFIER = "";
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator xros xrsimulator";
SUPPORTS_XR_DESIGNED_FOR_IPHONE_IPAD = NO;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2,7";
+ SWIFT_OBJC_BRIDGING_HEADER = "llama.swiftui/Bridging/llama_swiftui-Bridging-Header.h";
};
name = Release;
};
diff --git a/examples/llama.swiftui/llama.swiftui/Bridging/FinetuneBridge.h b/examples/llama.swiftui/llama.swiftui/Bridging/FinetuneBridge.h
new file mode 100644
index 000000000000..ff0d469e58bb
--- /dev/null
+++ b/examples/llama.swiftui/llama.swiftui/Bridging/FinetuneBridge.h
@@ -0,0 +1,55 @@
+#pragma once
+
+#include
+#include
+#include
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Error codes returned by llama_swift_run_lora_finetune
+enum llama_swift_finetune_error {
+ LLAMA_SWIFT_FINETUNE_OK = 0,
+ LLAMA_SWIFT_FINETUNE_ERROR_INVALID_ARGUMENT = 1,
+ LLAMA_SWIFT_FINETUNE_ERROR_MODEL_LOAD = 2,
+ LLAMA_SWIFT_FINETUNE_ERROR_CONTEXT_CREATE = 3,
+ LLAMA_SWIFT_FINETUNE_ERROR_DATASET = 4,
+ LLAMA_SWIFT_FINETUNE_ERROR_TRAINING_INIT = 5,
+ LLAMA_SWIFT_FINETUNE_ERROR_SAVE = 6,
+};
+
+struct llama_swift_finetune_options {
+ int32_t n_ctx;
+ int32_t n_threads;
+ int32_t n_batch;
+ int32_t n_ubatch;
+ int32_t epochs;
+ int32_t lora_rank;
+ float lora_alpha;
+ float learning_rate;
+ float val_split;
+ uint32_t target_modules;
+ int32_t seed;
+ bool flash_attn;
+ int32_t n_gpu_layers;
+
+ int32_t checkpoint_save_steps;
+ const char* checkpoint_save_dir;
+ const char* resume_from_checkpoint;
+ bool auto_resume;
+};
+
+typedef void (*llama_swift_finetune_log_callback)(const char * message, void * user_data);
+
+enum llama_swift_finetune_error llama_swift_run_lora_finetune(
+ const char * model_path,
+ const char * dataset_path,
+ const char * output_adapter_path,
+ const struct llama_swift_finetune_options * options,
+ llama_swift_finetune_log_callback logger,
+ void * user_data);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/examples/llama.swiftui/llama.swiftui/Bridging/FinetuneBridge.mm b/examples/llama.swiftui/llama.swiftui/Bridging/FinetuneBridge.mm
new file mode 100644
index 000000000000..ee15350adff4
--- /dev/null
+++ b/examples/llama.swiftui/llama.swiftui/Bridging/FinetuneBridge.mm
@@ -0,0 +1,861 @@
+#import "FinetuneBridge.h"
+
+#import
+#import
+#import
+#import
+
+#if defined(__APPLE__)
+#include
+#endif
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace {
+
+#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE
+constexpr bool kIsAppleMobileGPU = true;
+#else
+constexpr bool kIsAppleMobileGPU = false;
+#endif
+
+struct Hms {
+ int64_t hours;
+ int64_t minutes;
+ int64_t seconds;
+};
+
+Hms microseconds_to_hms(int64_t microseconds) {
+ if (microseconds < 0) {
+ microseconds = 0;
+ }
+
+ int64_t total_seconds = microseconds / 1000000;
+ Hms result{};
+ result.hours = total_seconds / 3600;
+ total_seconds -= result.hours * 3600;
+ result.minutes = total_seconds / 60;
+ total_seconds -= result.minutes * 60;
+ result.seconds = total_seconds;
+ return result;
+}
+
+struct Logger {
+ llama_swift_finetune_log_callback callback;
+ void * user;
+
+ void log(const std::string & message) const {
+ if (callback) {
+ callback(message.c_str(), user);
+ }
+ }
+
+ void logf(const char * fmt, ...) const {
+ if (!callback) {
+ return;
+ }
+
+ va_list args;
+ va_start(args, fmt);
+ std::array stack_buffer;
+ int written = vsnprintf(stack_buffer.data(), stack_buffer.size(), fmt, args);
+ va_end(args);
+
+ if (written < 0) {
+ return;
+ }
+
+ if (written < static_cast(stack_buffer.size())) {
+ callback(stack_buffer.data(), user);
+ return;
+ }
+
+ std::vector heap_buffer(static_cast(written + 1));
+ va_start(args, fmt);
+ vsnprintf(heap_buffer.data(), heap_buffer.size(), fmt, args);
+ va_end(args);
+
+ callback(heap_buffer.data(), user);
+ }
+};
+
+struct checkpoint_metadata {
+ int32_t epoch;
+ int32_t lora_rank;
+ float lora_alpha;
+ uint32_t target_modules;
+};
+
+struct checkpoint_callback_data {
+ llama_context* ctx;
+ llama_adapter_lora* adapter;
+ int32_t checkpoint_save_steps;
+ std::string checkpoint_save_dir;
+ int64_t global_step;
+ int64_t initial_step;
+ int32_t current_epoch;
+ int32_t lora_rank;
+ float lora_alpha;
+ uint32_t target_modules;
+ const Logger* logger;
+};
+
+static checkpoint_callback_data* g_checkpoint_data = nullptr;
+
+static std::string get_checkpoint_filename(const std::string& checkpoint_dir, int64_t step) {
+ std::ostringstream oss;
+ oss << checkpoint_dir << "/checkpoint_step_" << std::setfill('0') << std::setw(8) << step;
+ return oss.str();
+}
+
+static std::string find_latest_checkpoint(const std::string& checkpoint_dir) {
+ if (!std::filesystem::exists(checkpoint_dir)) {
+ return "";
+ }
+
+ std::string latest_checkpoint;
+ int64_t latest_step = -1;
+
+ for (const auto& entry : std::filesystem::directory_iterator(checkpoint_dir)) {
+ if (entry.is_directory()) {
+ std::string dirname = entry.path().filename().string();
+ if (dirname.find("checkpoint_step_") == 0 && dirname.size() >= 16) {
+ std::string step_str = dirname.substr(16, 8);
+ try {
+ int64_t step = std::stoll(step_str);
+ if (step > latest_step) {
+ latest_step = step;
+ latest_checkpoint = entry.path().string();
+ }
+ } catch (const std::exception&) {
+ continue;
+ }
+ }
+ }
+ }
+
+ return latest_checkpoint;
+}
+
+static bool save_checkpoint(
+ llama_context* ctx,
+ llama_adapter_lora* adapter,
+ const checkpoint_metadata& meta,
+ const std::string& checkpoint_dir,
+ const Logger& logger) {
+
+ try {
+ std::filesystem::create_directories(checkpoint_dir);
+ } catch (const std::exception& e) {
+ logger.logf("Failed to create checkpoint directory: %s\n", e.what());
+ return false;
+ }
+
+ std::string adapter_path = checkpoint_dir + "/model.gguf";
+ if (!llama_lora_save_adapter(adapter, adapter_path.c_str(), llama_get_model(ctx))) {
+ logger.logf("Failed to save LoRA adapter to %s\n", adapter_path.c_str());
+ return false;
+ }
+
+ std::string optimizer_path = checkpoint_dir + "/optimizer.gguf";
+ if (!llama_opt_save_state(ctx, optimizer_path.c_str())) {
+ logger.logf("Failed to save optimizer state to %s\n", optimizer_path.c_str());
+ return false;
+ }
+
+ std::string meta_path = checkpoint_dir + "/metadata.txt";
+ std::ofstream meta_file(meta_path);
+ if (meta_file.is_open()) {
+ meta_file << "epoch=" << meta.epoch << "\n";
+ meta_file << "lora_rank=" << meta.lora_rank << "\n";
+ meta_file << "lora_alpha=" << meta.lora_alpha << "\n";
+ meta_file << "target_modules=" << meta.target_modules << "\n";
+ meta_file.close();
+ } else {
+ logger.logf("Failed to save checkpoint metadata to %s\n", meta_path.c_str());
+ return false;
+ }
+
+ logger.logf("Checkpoint saved to %s\n", checkpoint_dir.c_str());
+ return true;
+}
+
+static bool load_checkpoint_metadata(const std::string& checkpoint_dir, checkpoint_metadata& metadata, const Logger& logger) {
+ std::string meta_path = checkpoint_dir + "/metadata.txt";
+
+ if (std::filesystem::exists(meta_path)) {
+ std::ifstream meta_file(meta_path);
+ if (meta_file.is_open()) {
+ std::string line;
+ while (std::getline(meta_file, line)) {
+ size_t eq_pos = line.find('=');
+ if (eq_pos != std::string::npos) {
+ std::string key = line.substr(0, eq_pos);
+ std::string value = line.substr(eq_pos + 1);
+
+ if (key == "epoch") {
+ metadata.epoch = std::stoi(value);
+ } else if (key == "lora_rank") {
+ metadata.lora_rank = std::stoi(value);
+ } else if (key == "lora_alpha") {
+ metadata.lora_alpha = std::stof(value);
+ } else if (key == "target_modules") {
+ metadata.target_modules = std::stoul(value);
+ }
+ }
+ }
+ meta_file.close();
+ logger.logf("Checkpoint metadata loaded successfully\n");
+ return true;
+ } else {
+ logger.logf("Failed to open checkpoint metadata file\n");
+ return false;
+ }
+ } else {
+ logger.logf("Checkpoint metadata file not found: %s\n", meta_path.c_str());
+ return false;
+ }
+}
+
+std::optional read_file(const char * path, std::string & error_message) {
+ std::ifstream stream(path, std::ios::binary);
+ if (!stream) {
+ error_message = "Unable to open dataset file";
+ return std::nullopt;
+ }
+
+ std::ostringstream buffer;
+ buffer << stream.rdbuf();
+ if (stream.fail() && !stream.eof()) {
+ error_message = "Failed reading dataset file";
+ return std::nullopt;
+ }
+
+ return buffer.str();
+}
+
+bool tokenize_dataset(const llama_model * model, const std::string & text, std::vector & tokens, std::string & error_message) {
+ const llama_vocab * vocab = llama_model_get_vocab(model);
+ if (!vocab) {
+ error_message = "Model vocabulary unavailable";
+ return false;
+ }
+
+ const int32_t text_len = static_cast(text.size());
+ int32_t capacity = text_len + 8;
+ capacity = std::max(capacity, 8);
+
+ tokens.resize(capacity);
+ int32_t n_tokens = llama_tokenize(vocab, text.c_str(), text_len, tokens.data(), static_cast(tokens.size()), /*add_bos=*/true, /*parse_special=*/false);
+ if (n_tokens == std::numeric_limits::min()) {
+ error_message = "Dataset too large to tokenize";
+ return false;
+ }
+ if (n_tokens < 0) {
+ tokens.resize(static_cast(-n_tokens));
+ n_tokens = llama_tokenize(vocab, text.c_str(), text_len, tokens.data(), static_cast(tokens.size()), /*add_bos=*/true, /*parse_special=*/false);
+ } else {
+ tokens.resize(static_cast(n_tokens));
+ }
+
+ if (n_tokens <= 0) {
+ error_message = "Dataset produced no tokens";
+ return false;
+ }
+
+ return true;
+}
+
+int32_t default_thread_count() {
+ unsigned int hw = std::thread::hardware_concurrency();
+ if (hw == 0) {
+ return 4;
+ }
+ if (hw <= 2) {
+ return 1;
+ }
+ return static_cast(hw - 1);
+}
+
+struct EpochLoggerState {
+ const Logger * logger = nullptr;
+ int32_t epoch_index = 0;
+ int32_t epoch_total = 0;
+ int64_t last_logged_train = -1;
+ int64_t last_logged_eval = -1;
+};
+
+static thread_local EpochLoggerState g_epoch_logger_state;
+
+struct EpochLoggerScope {
+ EpochLoggerScope(const Logger & logger, int32_t epoch_index, int32_t epoch_total) {
+ g_epoch_logger_state.logger = &logger;
+ g_epoch_logger_state.epoch_index = epoch_index;
+ g_epoch_logger_state.epoch_total = epoch_total;
+ g_epoch_logger_state.last_logged_train = -1;
+ g_epoch_logger_state.last_logged_eval = -1;
+ }
+
+ ~EpochLoggerScope() {
+ g_epoch_logger_state.logger = nullptr;
+ }
+};
+
+static void finetune_epoch_progress_callback(
+ bool train,
+ ggml_opt_context_t opt_ctx,
+ ggml_opt_dataset_t,
+ ggml_opt_result_t result,
+ int64_t ibatch,
+ int64_t ibatch_max,
+ int64_t t_start_us) {
+
+ EpochLoggerState & state = g_epoch_logger_state;
+ if (!state.logger || ibatch_max <= 0) {
+ return;
+ }
+
+ int64_t & last_logged = train ? state.last_logged_train : state.last_logged_eval;
+ if (ibatch <= last_logged) {
+ return;
+ }
+ last_logged = ibatch;
+
+ const int64_t total_batches = std::max(ibatch_max, int64_t(1));
+ const int64_t clamped_batch = std::clamp(ibatch, int64_t(1), total_batches);
+
+ const ggml_tensor * input_tensor = ggml_opt_inputs(opt_ctx);
+ const int64_t batch_size = input_tensor ? input_tensor->ne[1] : 0;
+ const int64_t data_processed = batch_size > 0 ? clamped_batch * batch_size : clamped_batch;
+ const int64_t data_total = batch_size > 0 ? total_batches * batch_size : total_batches;
+
+ double loss = 0.0;
+ double loss_unc = 0.0;
+ ggml_opt_result_loss(result, &loss, &loss_unc);
+ if (!std::isfinite(loss)) {
+ loss = 0.0;
+ }
+ if (!std::isfinite(loss_unc)) {
+ loss_unc = 0.0;
+ }
+
+ double accuracy = 0.0;
+ double accuracy_unc = 0.0;
+ ggml_opt_result_accuracy(result, &accuracy, &accuracy_unc);
+ if (!std::isfinite(accuracy)) {
+ accuracy = 0.0;
+ }
+ if (!std::isfinite(accuracy_unc)) {
+ accuracy_unc = 0.0;
+ }
+
+ int64_t elapsed_us = 0;
+ if (t_start_us > 0) {
+ const int64_t now_us = ggml_time_us();
+ if (now_us >= t_start_us) {
+ elapsed_us = now_us - t_start_us;
+ }
+ }
+
+ int64_t eta_us = 0;
+ if (clamped_batch > 0 && total_batches > clamped_batch) {
+ eta_us = (elapsed_us * (total_batches - clamped_batch)) / clamped_batch;
+ }
+
+ const Hms elapsed_hms = microseconds_to_hms(elapsed_us);
+ const Hms eta_hms = microseconds_to_hms(eta_us);
+
+ const double progress = (static_cast(clamped_batch) * 100.0) / static_cast(total_batches);
+
+ state.logger->logf(
+ " [epoch %d/%d][%s] step %lld/%lld data=%lld/%lld loss=%.6f±%.6f acc=%.2f±%.2f%% "
+ "t=%02lld:%02lld:%02lld ETA=%02lld:%02lld:%02lld (%.1f%%)\n",
+ state.epoch_index + 1,
+ state.epoch_total,
+ train ? "train" : "eval",
+ static_cast(clamped_batch),
+ static_cast(total_batches),
+ static_cast(data_processed),
+ static_cast(data_total),
+ loss,
+ loss_unc,
+ 100.0 * accuracy,
+ 100.0 * accuracy_unc,
+ static_cast(elapsed_hms.hours),
+ static_cast(elapsed_hms.minutes),
+ static_cast(elapsed_hms.seconds),
+ static_cast(eta_hms.hours),
+ static_cast(eta_hms.minutes),
+ static_cast(eta_hms.seconds),
+ progress);
+}
+
+static void checkpoint_progress_callback(
+ bool train,
+ ggml_opt_context_t opt_ctx,
+ ggml_opt_dataset_t dataset,
+ ggml_opt_result_t result,
+ int64_t ibatch,
+ int64_t ibatch_max,
+ int64_t t_start_us) {
+
+ finetune_epoch_progress_callback(train, opt_ctx, dataset, result, ibatch, ibatch_max, t_start_us);
+
+ if (!train) {
+ return;
+ }
+
+ checkpoint_callback_data* cb_data = g_checkpoint_data;
+ if (!cb_data) {
+ return;
+ }
+
+ if (cb_data->checkpoint_save_steps <= 0) {
+ return;
+ }
+
+ cb_data->global_step++;
+
+ if (cb_data->global_step % cb_data->checkpoint_save_steps == 0) {
+ if (!cb_data->ctx) {
+ if (cb_data->logger) {
+ cb_data->logger->logf("ERROR: Context is null in checkpoint callback!\n");
+ }
+ return;
+ }
+
+ if (!cb_data->adapter) {
+ if (cb_data->logger) {
+ cb_data->logger->logf("ERROR: LoRA adapter is null in checkpoint callback!\n");
+ }
+ return;
+ }
+
+ checkpoint_metadata meta = {
+ /*epoch =*/ cb_data->current_epoch,
+ /*lora_rank =*/ cb_data->lora_rank,
+ /*lora_alpha =*/ cb_data->lora_alpha,
+ /*target_modules =*/ cb_data->target_modules,
+ };
+
+ std::string checkpoint_path = get_checkpoint_filename(cb_data->checkpoint_save_dir, cb_data->global_step);
+
+ if (cb_data->logger) {
+ cb_data->logger->logf("Saving checkpoint at step %lld to %s\n",
+ (long long)cb_data->global_step, checkpoint_path.c_str());
+ }
+
+ if (!save_checkpoint(cb_data->ctx, cb_data->adapter, meta, checkpoint_path, *cb_data->logger)) {
+ if (cb_data->logger) {
+ cb_data->logger->logf("ERROR: Failed to save checkpoint at step %lld\n",
+ (long long)cb_data->global_step);
+ }
+ }
+ }
+}
+
+} // namespace
+
+extern "C" enum llama_swift_finetune_error llama_swift_run_lora_finetune(
+ const char * model_path,
+ const char * dataset_path,
+ const char * output_adapter_path,
+ const struct llama_swift_finetune_options * options,
+ llama_swift_finetune_log_callback logger_callback,
+ void * user_data) {
+
+ Logger logger{logger_callback, user_data};
+
+ if (!model_path || !dataset_path || !output_adapter_path) {
+ logger.log("Invalid arguments supplied to finetune request\n");
+ return LLAMA_SWIFT_FINETUNE_ERROR_INVALID_ARGUMENT;
+ }
+
+ llama_swift_finetune_options opts{};
+ if (options) {
+ opts = *options;
+ }
+
+ if (opts.n_ctx <= 0) {
+ opts.n_ctx = 256;
+ }
+ if (opts.n_threads <= 0) {
+ opts.n_threads = default_thread_count();
+ }
+ if (opts.n_batch <= 0) {
+ opts.n_batch = 256;
+ }
+ if (opts.n_ubatch <= 0) {
+ opts.n_ubatch = opts.n_batch;
+ }
+ if (opts.epochs <= 0) {
+ opts.epochs = 2;
+ }
+ if (opts.lora_rank <= 0) {
+ opts.lora_rank = 8;
+ }
+ if (opts.lora_alpha <= 0.0f) {
+ opts.lora_alpha = 16.0f;
+ }
+ if (opts.learning_rate <= 0.0f) {
+ opts.learning_rate = 1e-5f;
+ }
+ if (opts.val_split < 0.0f) {
+ opts.val_split = 0.0f;
+ }
+ if (opts.val_split >= 1.0f) {
+ opts.val_split = 0.0f;
+ }
+ if (opts.target_modules == 0) {
+ opts.target_modules = LLAMA_LORA_TARGET_ATTN_Q |
+ LLAMA_LORA_TARGET_ATTN_K |
+ LLAMA_LORA_TARGET_ATTN_V |
+ LLAMA_LORA_TARGET_ATTN_O;
+ }
+ if (opts.n_gpu_layers < 0) {
+ const char * env_ngl = std::getenv("LLAMA_SWIFT_FINETUNE_NGL");
+ if (env_ngl && *env_ngl) {
+ char * endptr = nullptr;
+ long parsed = std::strtol(env_ngl, &endptr, 10);
+ if (endptr != env_ngl && parsed >= 0 && parsed <= std::numeric_limits::max()) {
+ opts.n_gpu_layers = static_cast(parsed);
+ logger.logf("Environment override: using n_gpu_layers=%d\n", opts.n_gpu_layers);
+ } else {
+ logger.logf("Ignoring invalid LLAMA_SWIFT_FINETUNE_NGL value '%s'\n", env_ngl);
+ opts.n_gpu_layers = 999;
+ }
+ } else {
+ opts.n_gpu_layers = 999;
+ }
+ }
+
+ // opts.n_gpu_layers = 0;
+
+ const bool gpu_available = opts.n_gpu_layers > 0 && llama_supports_gpu_offload();
+ if (gpu_available) {
+ const int32_t max_gpu_batch = kIsAppleMobileGPU ? 64 : 256;
+ const int32_t max_gpu_ubatch = kIsAppleMobileGPU ? 32 : 128;
+
+ if (opts.n_batch > max_gpu_batch) {
+ logger.logf("Clamping batch size from %d to %d for on-device GPU stability\n", opts.n_batch, max_gpu_batch);
+ opts.n_batch = max_gpu_batch;
+ }
+
+ if (opts.n_ubatch > max_gpu_ubatch) {
+ logger.logf("Clamping micro-batch size from %d to %d for on-device GPU stability\n", opts.n_ubatch, max_gpu_ubatch);
+ opts.n_ubatch = max_gpu_ubatch;
+ }
+
+ if (opts.n_ubatch > opts.n_batch) {
+ logger.logf("Adjusting micro-batch size to not exceed batch size (%d -> %d)\n", opts.n_ubatch, opts.n_batch);
+ opts.n_ubatch = opts.n_batch;
+ }
+ }
+
+ llama_backend_init();
+
+ llama_model_params model_params = llama_model_default_params();
+ model_params.n_gpu_layers = opts.n_gpu_layers;
+ model_params.use_mmap = false;
+ model_params.use_mlock = false;
+
+ logger.logf("Loading model from %s\n", model_path);
+ llama_model * model = llama_model_load_from_file(model_path, model_params);
+ if (!model) {
+ logger.log("Failed to load model for finetuning\n");
+ return LLAMA_SWIFT_FINETUNE_ERROR_MODEL_LOAD;
+ }
+
+ llama_context_params ctx_params = llama_context_default_params();
+ ctx_params.n_ctx = opts.n_ctx;
+ ctx_params.n_batch = opts.n_batch;
+ ctx_params.n_ubatch = opts.n_ubatch;
+ ctx_params.n_threads = opts.n_threads;
+ ctx_params.n_threads_batch = opts.n_threads;
+ ctx_params.flash_attn_type = opts.flash_attn ? LLAMA_FLASH_ATTN_TYPE_ENABLED : LLAMA_FLASH_ATTN_TYPE_DISABLED;
+
+ if (opts.seed != 0) {
+ logger.log("Warning: custom RNG seed requested but not supported by bundled llama.framework.\n");
+ }
+
+ logger.logf("Creating training context (n_ctx=%d, n_threads=%d, batch=%d, ubatch=%d, ngl=%d)\n",
+ ctx_params.n_ctx, ctx_params.n_threads, ctx_params.n_batch, ctx_params.n_ubatch, opts.n_gpu_layers);
+
+ llama_context * ctx = llama_init_from_model(model, ctx_params);
+ if (!ctx) {
+ logger.log("Failed to create llama context for finetuning\n");
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_CONTEXT_CREATE;
+ }
+
+ std::string error_message;
+ auto dataset_text = read_file(dataset_path, error_message);
+ if (!dataset_text.has_value()) {
+ logger.logf("%s: %s\n", error_message.c_str(), dataset_path);
+ llama_free(ctx);
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_DATASET;
+ }
+
+ std::vector tokens;
+ if (!tokenize_dataset(model, dataset_text.value(), tokens, error_message)) {
+ logger.logf("Tokenization failed: %s\n", error_message.c_str());
+ llama_free(ctx);
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_DATASET;
+ }
+
+ const int64_t ne_datapoint = llama_n_ctx(ctx);
+ const int64_t stride = std::max(1, ne_datapoint / 2);
+ if (tokens.size() < static_cast(ne_datapoint + 1)) {
+ logger.log("Dataset is too short for the selected context size\n");
+ llama_free(ctx);
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_DATASET;
+ }
+
+ const int64_t ndata = (static_cast(tokens.size()) - ne_datapoint - 1) / stride;
+ if (ndata <= 0) {
+ logger.log("Tokenized dataset produced no training samples\n");
+ llama_free(ctx);
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_DATASET;
+ }
+
+ ggml_opt_dataset_t dataset = ggml_opt_dataset_init(
+ GGML_TYPE_I32,
+ GGML_TYPE_I32,
+ ne_datapoint,
+ ne_datapoint,
+ ndata,
+ /*ndata_shard=*/1);
+
+ llama_token * data_ptr = reinterpret_cast(ggml_opt_dataset_data(dataset)->data);
+ llama_token * labels_ptr = reinterpret_cast(ggml_opt_dataset_labels(dataset)->data);
+
+ for (int64_t i = 0; i < ndata; ++i) {
+ std::memcpy(data_ptr + i * ne_datapoint, tokens.data() + i * stride, static_cast(ne_datapoint) * sizeof(llama_token));
+ std::memcpy(labels_ptr + i * ne_datapoint, tokens.data() + i * stride + 1, static_cast(ne_datapoint) * sizeof(llama_token));
+ }
+
+ logger.logf("Prepared dataset with %lld sequences (stride=%lld)\n", static_cast(ndata), static_cast(stride));
+
+ llama_lora_training_params lora_params{
+ /*target_modules =*/ opts.target_modules,
+ /*rank =*/ opts.lora_rank,
+ /*alpha =*/ opts.lora_alpha,
+ /*dropout =*/ 0.0f,
+ /*init_std =*/ 0.02f,
+ };
+
+ llama_adapter_lora * adapter = llama_lora_training_init(ctx, model, &lora_params);
+ if (!adapter) {
+ logger.log("Failed to initialize LoRA training\n");
+ ggml_opt_dataset_free(dataset);
+ llama_free(ctx);
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_TRAINING_INIT;
+ }
+
+ bool checkpoint_loaded = false;
+ int32_t start_epoch = 0;
+ int64_t start_step = 0;
+ std::string checkpoint_dir = opts.checkpoint_save_dir ? opts.checkpoint_save_dir : "./checkpoints";
+ std::string resume_from_checkpoint;
+
+ if (opts.auto_resume) {
+ logger.logf("Auto-resume enabled, searching for latest checkpoint in %s\n", checkpoint_dir.c_str());
+ resume_from_checkpoint = find_latest_checkpoint(checkpoint_dir);
+ if (!resume_from_checkpoint.empty()) {
+ logger.logf("Found latest checkpoint: %s\n", resume_from_checkpoint.c_str());
+ } else {
+ logger.logf("No existing checkpoint found, starting from scratch\n");
+ }
+ } else if (opts.resume_from_checkpoint && opts.resume_from_checkpoint[0] != '\0') {
+ resume_from_checkpoint = opts.resume_from_checkpoint;
+ logger.logf("Resuming from specified checkpoint: %s\n", resume_from_checkpoint.c_str());
+ }
+
+ if (!resume_from_checkpoint.empty()) {
+ checkpoint_metadata checkpoint_meta{};
+ if (load_checkpoint_metadata(resume_from_checkpoint, checkpoint_meta, logger)) {
+ if (checkpoint_meta.lora_rank != opts.lora_rank) {
+ logger.logf("Warning: Checkpoint LoRA rank (%d) doesn't match current rank (%d)\n",
+ checkpoint_meta.lora_rank, opts.lora_rank);
+ }
+ if (checkpoint_meta.lora_alpha != opts.lora_alpha) {
+ logger.logf("Warning: Checkpoint LoRA alpha (%.3f) doesn't match current alpha (%.3f)\n",
+ checkpoint_meta.lora_alpha, opts.lora_alpha);
+ }
+ if (checkpoint_meta.target_modules != opts.target_modules) {
+ logger.logf("Warning: Checkpoint target_modules doesn't match current target_modules\n");
+ }
+
+ start_epoch = checkpoint_meta.epoch;
+ checkpoint_loaded = true;
+ } else {
+ logger.logf("Failed to load checkpoint metadata, starting from scratch\n");
+ }
+ }
+
+ std::string optimizer_checkpoint_path;
+ if (checkpoint_loaded && !resume_from_checkpoint.empty()) {
+ optimizer_checkpoint_path = resume_from_checkpoint + "/optimizer.gguf";
+ }
+
+ ggml_opt_optimizer_params optimizer_params = ggml_opt_get_default_optimizer_params(nullptr);
+ optimizer_params.adamw.alpha = opts.learning_rate;
+
+ llama_opt_params opt_params{
+ /*n_ctx_train =*/ 0,
+ /*param_filter =*/ llama_opt_param_filter_lora,
+ /*param_filter_ud =*/ nullptr,
+ /*get_opt_pars =*/ ggml_opt_get_constant_optimizer_params,
+ /*get_opt_pars_ud =*/ &optimizer_params,
+ /*optimizer_type =*/ GGML_OPT_OPTIMIZER_TYPE_ADAMW,
+ /*checkpoint_path =*/ checkpoint_loaded ? optimizer_checkpoint_path.c_str() : nullptr,
+ /*load_optimizer_state =*/ checkpoint_loaded,
+ /*assistant_loss_only =*/ false,
+ };
+
+ llama_opt_init(ctx, model, opt_params);
+
+ ggml_opt_result_t result_train = ggml_opt_result_init();
+ ggml_opt_result_t result_eval = ggml_opt_result_init();
+
+ const int64_t total_samples = ggml_opt_dataset_ndata(dataset);
+ int64_t idata_split = total_samples;
+ if (opts.val_split > 0.0f && opts.val_split < 1.0f) {
+ const double train_fraction = std::clamp(1.0 - static_cast(opts.val_split), 0.0, 1.0);
+ int64_t proposed = static_cast(std::llround(train_fraction * static_cast(total_samples)));
+ proposed = std::clamp(proposed, 1, std::max(total_samples - 1, 1));
+ idata_split = proposed;
+ }
+
+ checkpoint_callback_data cb_data{};
+ cb_data.ctx = ctx;
+ cb_data.adapter = adapter;
+ cb_data.checkpoint_save_steps = opts.checkpoint_save_steps;
+ cb_data.checkpoint_save_dir = checkpoint_dir;
+ cb_data.global_step = start_step;
+ cb_data.initial_step = start_step;
+ cb_data.current_epoch = start_epoch;
+ cb_data.lora_rank = opts.lora_rank;
+ cb_data.lora_alpha = opts.lora_alpha;
+ cb_data.target_modules = opts.target_modules;
+ cb_data.logger = &logger;
+
+ g_checkpoint_data = &cb_data;
+
+ ggml_opt_epoch_callback train_callback = opts.checkpoint_save_steps > 0
+ ? checkpoint_progress_callback
+ : finetune_epoch_progress_callback;
+
+ ggml_opt_epoch_callback eval_callback = (idata_split < total_samples)
+ ? finetune_epoch_progress_callback
+ : nullptr;
+
+ if (opts.checkpoint_save_steps > 0) {
+ logger.logf("Checkpointing enabled, saving every %d steps to %s\n",
+ opts.checkpoint_save_steps, checkpoint_dir.c_str());
+ } else {
+ logger.logf("Checkpointing disabled\n");
+ }
+
+ logger.logf("Starting LoRA finetuning for %d epoch(s)\n", opts.epochs);
+
+ for (int32_t epoch = start_epoch; epoch < opts.epochs; ++epoch) {
+ logger.logf("Epoch %d/%d\n", epoch + 1, opts.epochs);
+ EpochLoggerScope epoch_scope(logger, epoch, opts.epochs);
+
+ cb_data.current_epoch = epoch;
+
+ int64_t resume_batch = 0;
+
+ llama_opt_epoch_resume(ctx,
+ dataset,
+ result_train,
+ result_eval,
+ idata_split,
+ train_callback,
+ eval_callback,
+ resume_batch);
+
+ double train_loss = 0.0;
+ double train_unc = 0.0;
+ ggml_opt_result_loss(result_train, &train_loss, &train_unc);
+ ggml_opt_result_reset(result_train);
+
+ if (idata_split < total_samples) {
+ double val_loss = 0.0;
+ double val_unc = 0.0;
+ ggml_opt_result_loss(result_eval, &val_loss, &val_unc);
+ ggml_opt_result_reset(result_eval);
+ logger.logf(" train loss: %.6f val loss: %.6f\n", train_loss, val_loss);
+ } else {
+ logger.logf(" train loss: %.6f\n", train_loss);
+ }
+ }
+
+ g_checkpoint_data = nullptr;
+
+ std::filesystem::path output_path(output_adapter_path);
+ if (output_path.has_parent_path()) {
+ std::error_code ec;
+ std::filesystem::create_directories(output_path.parent_path(), ec);
+ if (ec) {
+ logger.logf("Failed to create output directory: %s (%d)\n", ec.message().c_str(), static_cast(ec.value()));
+ ggml_opt_result_free(result_train);
+ ggml_opt_result_free(result_eval);
+ ggml_opt_dataset_free(dataset);
+ llama_adapter_lora_free(adapter);
+ llama_free(ctx);
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_SAVE;
+ }
+ }
+
+ logger.logf("Saving LoRA adapter to %s\n", output_adapter_path);
+ if (!llama_lora_save_adapter(adapter, output_adapter_path, model)) {
+ logger.log("Failed to save LoRA adapter\n");
+ ggml_opt_result_free(result_train);
+ ggml_opt_result_free(result_eval);
+ ggml_opt_dataset_free(dataset);
+ llama_adapter_lora_free(adapter);
+ llama_free(ctx);
+ llama_model_free(model);
+ return LLAMA_SWIFT_FINETUNE_ERROR_SAVE;
+ }
+
+ std::error_code size_ec;
+ const auto file_size = std::filesystem::file_size(output_path, size_ec);
+ if (!size_ec) {
+ logger.logf("Saved adapter size: %.2f MB\n", static_cast(file_size) / (1024.0 * 1024.0));
+ }
+
+ ggml_opt_result_free(result_train);
+ ggml_opt_result_free(result_eval);
+ ggml_opt_dataset_free(dataset);
+ llama_adapter_lora_free(adapter);
+ llama_free(ctx);
+ llama_model_free(model);
+
+ logger.log("LoRA finetuning completed successfully\n");
+ return LLAMA_SWIFT_FINETUNE_OK;
+}
diff --git a/examples/llama.swiftui/llama.swiftui/Bridging/llama_swiftui-Bridging-Header.h b/examples/llama.swiftui/llama.swiftui/Bridging/llama_swiftui-Bridging-Header.h
new file mode 100644
index 000000000000..74ef699ec2e0
--- /dev/null
+++ b/examples/llama.swiftui/llama.swiftui/Bridging/llama_swiftui-Bridging-Header.h
@@ -0,0 +1 @@
+#import "FinetuneBridge.h"
diff --git a/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift b/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift
index b8f6a31d582c..111945eaf816 100644
--- a/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift
+++ b/examples/llama.swiftui/llama.swiftui/Models/LlamaState.swift
@@ -8,33 +8,235 @@ struct Model: Identifiable {
var status: String?
}
+struct Dataset: Identifiable, Equatable {
+ var id = UUID()
+ var name: String
+ var url: String
+ var filename: String
+ var status: String?
+}
+
@MainActor
class LlamaState: ObservableObject {
@Published var messageLog = ""
@Published var cacheCleared = false
@Published var downloadedModels: [Model] = []
@Published var undownloadedModels: [Model] = []
+ @Published var selectedModelFilename: String?
+ @Published var isFinetuning = false
+ @Published var downloadedDatasets: [Dataset] = []
+ @Published var availableDatasets: [Dataset] = []
+ @Published var selectedDatasetFilename: String?
+ @Published var optionNGpuLayers: Int = -1 {
+ didSet {
+ let clamped = max(-1, min(optionNGpuLayers, 256))
+ if optionNGpuLayers != clamped {
+ optionNGpuLayers = clamped
+ return
+ }
+ UserDefaults.standard.set(optionNGpuLayers, forKey: runtimeNglKey)
+ if optionNGpuLayers != oldValue {
+ reloadModelWithCurrentOptionsIfPossible()
+ }
+ }
+ }
+
+ @Published var optionContextLength: Int = 2048 {
+ didSet {
+ let clamped = max(32, min(optionContextLength, 8192))
+ if optionContextLength != clamped {
+ optionContextLength = clamped
+ return
+ }
+ UserDefaults.standard.set(optionContextLength, forKey: runtimeContextKey)
+ if optionContextLength != oldValue {
+ reloadModelWithCurrentOptionsIfPossible()
+ }
+ }
+ }
+
+ @Published var optionSeed: Int = 42 {
+ didSet {
+ let clamped = max(0, min(optionSeed, 1_000_000))
+ if optionSeed != clamped {
+ optionSeed = clamped
+ return
+ }
+ UserDefaults.standard.set(optionSeed, forKey: runtimeSeedKey)
+ if optionSeed != oldValue {
+ applySamplerUpdate()
+ }
+ }
+ }
+
+ @Published var optionTemperature: Double = 0.0 {
+ didSet {
+ let clamped = min(max(optionTemperature, 0.0), 0.0)
+ if abs(optionTemperature - clamped) > 0.0001 {
+ optionTemperature = clamped
+ return
+ }
+ UserDefaults.standard.set(optionTemperature, forKey: runtimeTempKey)
+ if abs(optionTemperature - oldValue) > 0.0001 {
+ applySamplerUpdate()
+ }
+ }
+ }
+
+ @Published var optionTopP: Double = 0.95 {
+ didSet {
+ let clamped = min(max(optionTopP, 0.01), 1.0)
+ if abs(optionTopP - clamped) > 0.0001 {
+ optionTopP = clamped
+ return
+ }
+ UserDefaults.standard.set(optionTopP, forKey: runtimeTopPKey)
+ if abs(optionTopP - oldValue) > 0.0001 {
+ applySamplerUpdate()
+ }
+ }
+ }
+
+ @Published var optionTopK: Int = 40 {
+ didSet {
+ let clamped = max(1, min(optionTopK, 500))
+ if optionTopK != clamped {
+ optionTopK = clamped
+ return
+ }
+ UserDefaults.standard.set(optionTopK, forKey: runtimeTopKKey)
+ if optionTopK != oldValue {
+ applySamplerUpdate()
+ }
+ }
+ }
+
+ @Published var optionFlashAttention: Bool = false {
+ didSet {
+ UserDefaults.standard.set(optionFlashAttention, forKey: runtimeFlashAttnKey)
+ if optionFlashAttention != oldValue {
+ reloadModelWithCurrentOptionsIfPossible()
+ }
+ }
+ }
+
+ @Published var optionSingleTurn: Bool = true {
+ didSet {
+ UserDefaults.standard.set(optionSingleTurn, forKey: runtimeSingleTurnKey)
+ }
+ }
+
let NS_PER_S = 1_000_000_000.0
private var llamaContext: LlamaContext?
+ private var currentModelURL: URL?
+ private let modelSelectionKey = "llama_swiftui_selected_model"
+ private let datasetSelectionKey = "llama_swiftui_selected_dataset"
+ private let allowedModelExtensions: Set = ["gguf", "ggml", "bin"]
+ private let runtimeNglKey = "llama_swiftui_runtime_ngl"
+ private let runtimeContextKey = "llama_swiftui_runtime_ctx"
+ private let runtimeSeedKey = "llama_swiftui_runtime_seed"
+ private let runtimeTempKey = "llama_swiftui_runtime_temp"
+ private let runtimeTopPKey = "llama_swiftui_runtime_top_p"
+ private let runtimeTopKKey = "llama_swiftui_runtime_top_k"
+ private let runtimeFlashAttnKey = "llama_swiftui_runtime_flash_attn"
+ private let runtimeSingleTurnKey = "llama_swiftui_runtime_single_turn"
+
+ private typealias FinetuneLogCallback = @convention(c) (UnsafePointer?, UnsafeMutableRawPointer?) -> Void
+
+ private static let finetuneLogCallback: FinetuneLogCallback = { messagePtr, userData in
+ guard let userData else { return }
+ let state = Unmanaged.fromOpaque(userData).takeUnretainedValue()
+ guard let messagePtr else { return }
+ let text = String(cString: messagePtr)
+ Task { @MainActor in
+ state.messageLog += text
+ }
+ }
private var defaultModelUrl: URL? {
Bundle.main.url(forResource: "ggml-model", withExtension: "gguf", subdirectory: "models")
// Bundle.main.url(forResource: "llama-2-7b-chat", withExtension: "Q2_K.gguf", subdirectory: "models")
}
+ private let finetuneDatasetFilename = "trump.txt"
+ private let defaultFinetuneDatasetURL = URL(string: "https://github.com/user-attachments/files/21859494/trump.txt")
+ private let rockTweetsDatasetFilename = "the-rock-tweets.txt"
+ private let rockTweetsDatasetURL = URL(string: "https://gist.githubusercontent.com/zoq/774ab751bb3599835362bd6b5b604044/raw/a0d58f5d4bcd46621f9a3112c6fee2c3142fc8a0/the-rock-tweets.txt")
+
+ private lazy var defaultDatasets: [Dataset] = {
+ var defaults: [Dataset] = []
+ if let url = defaultFinetuneDatasetURL {
+ defaults.append(Dataset(name: "Sample Trump Interview", url: url.absoluteString, filename: finetuneDatasetFilename, status: "download"))
+ }
+ if let rockTweetsURL = rockTweetsDatasetURL {
+ defaults.append(Dataset(name: "Sample The Rock Tweets", url: rockTweetsURL.absoluteString, filename: rockTweetsDatasetFilename, status: "download"))
+ }
+ return defaults
+ }()
+
+ private let disallowedDatasetExtensions: Set = ["gguf", "ggml", "bin"]
+
init() {
+ let defaults = UserDefaults.standard
+ if let stored = defaults.object(forKey: runtimeNglKey) as? Int {
+ optionNGpuLayers = max(-1, min(stored, 256))
+ }
+ if let stored = defaults.object(forKey: runtimeContextKey) as? Int, stored >= 32 {
+ optionContextLength = min(stored, 8192)
+ }
+ if let stored = defaults.object(forKey: runtimeSeedKey) as? Int, stored >= 0 {
+ optionSeed = min(stored, 1_000_000)
+ }
+ if let stored = defaults.object(forKey: runtimeTempKey) as? Double {
+ optionTemperature = min(max(stored, 0.0), 0.0)
+ }
+ if let stored = defaults.object(forKey: runtimeTopPKey) as? Double {
+ optionTopP = min(max(stored, 0.01), 1.0)
+ }
+ if let stored = defaults.object(forKey: runtimeTopKKey) as? Int, stored >= 1 {
+ optionTopK = min(stored, 500)
+ }
+ if let stored = defaults.object(forKey: runtimeFlashAttnKey) as? Bool {
+ optionFlashAttention = stored
+ }
+ if let stored = defaults.object(forKey: runtimeSingleTurnKey) as? Bool {
+ optionSingleTurn = stored
+ }
+
loadModelsFromDisk()
loadDefaultModels()
+ selectedModelFilename = UserDefaults.standard.string(forKey: modelSelectionKey)
+ ensureSelectedModelIsValid()
+ loadDatasetsFromDisk()
+ loadDefaultDatasets()
+ selectedDatasetFilename = UserDefaults.standard.string(forKey: datasetSelectionKey)
+ ensureSelectedDatasetIsValid()
}
private func loadModelsFromDisk() {
+ downloadedModels.removeAll { model in
+ let fileURL = modelFileURL(for: model.filename)
+ return !FileManager.default.fileExists(atPath: fileURL.path)
+ }
+
do {
let documentsURL = getDocumentsDirectory()
- let modelURLs = try FileManager.default.contentsOfDirectory(at: documentsURL, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants])
- for modelURL in modelURLs {
- let modelName = modelURL.deletingPathExtension().lastPathComponent
- downloadedModels.append(Model(name: modelName, url: "", filename: modelURL.lastPathComponent, status: "downloaded"))
+ let modelURLs = try FileManager.default.contentsOfDirectory(
+ at: documentsURL,
+ includingPropertiesForKeys: nil,
+ options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants]
+ )
+
+ for modelURL in modelURLs where !modelURL.hasDirectoryPath {
+ let ext = modelURL.pathExtension.lowercased()
+ guard allowedModelExtensions.contains(ext) else { continue }
+ let filename = modelURL.lastPathComponent
+ guard !downloadedModels.contains(where: { $0.filename == filename }) else { continue }
+
+ let displayName = modelURL.deletingPathExtension().lastPathComponent
+ downloadedModels.append(Model(name: displayName, url: "", filename: filename, status: "downloaded"))
}
+ downloadedModels.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
} catch {
print("Error loading models from disk: \(error)")
}
@@ -48,14 +250,217 @@ class LlamaState: ObservableObject {
}
for model in defaultModels {
- let fileURL = getDocumentsDirectory().appendingPathComponent(model.filename)
+ let fileURL = modelFileURL(for: model.filename)
if FileManager.default.fileExists(atPath: fileURL.path) {
+ continue
+ }
+ guard !undownloadedModels.contains(where: { $0.filename == model.filename }) else { continue }
+
+ var undownloadedModel = model
+ undownloadedModel.status = "download"
+ undownloadedModels.append(undownloadedModel)
+ }
+
+ undownloadedModels.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ }
- } else {
- var undownloadedModel = model
- undownloadedModel.status = "download"
- undownloadedModels.append(undownloadedModel)
+ private func ensureSelectedModelIsValid() {
+ guard let selectedFilename = selectedModelFilename else { return }
+ let fileURL = modelFileURL(for: selectedFilename)
+ let bundleMatch = defaultModelUrl?.lastPathComponent == selectedFilename
+ if FileManager.default.fileExists(atPath: fileURL.path) || bundleMatch {
+ return
+ }
+ updateSelectedModel(filename: nil, name: nil, announce: false)
+ }
+
+ private func updateSelectedModel(filename: String?, name: String?, announce: Bool) {
+ if let filename {
+ selectedModelFilename = filename
+ UserDefaults.standard.set(filename, forKey: modelSelectionKey)
+ if announce {
+ let displayName = name ?? filename
+ messageLog += "Ready to use model \(displayName).\n"
}
+ } else {
+ selectedModelFilename = nil
+ UserDefaults.standard.removeObject(forKey: modelSelectionKey)
+ }
+ }
+
+ private func runtimeOptions() -> LlamaRuntimeOptions {
+ let context = Int32(optionContextLength)
+ let ngl = optionNGpuLayers >= 0 ? Int32(optionNGpuLayers) : -1
+ let seedValue = optionSeed < 0 ? 0 : UInt32(optionSeed)
+ let temperature = Float(min(max(optionTemperature, 0.0), 0.0))
+ let topP = Float(min(max(optionTopP, 0.01), 1.0))
+ let topK = Int32(max(1, optionTopK))
+
+ return LlamaRuntimeOptions(
+ contextLength: context,
+ nGpuLayers: ngl,
+ seed: seedValue,
+ temperature: temperature,
+ topP: topP,
+ topK: topK,
+ flashAttention: optionFlashAttention
+ )
+ }
+
+ private func reloadModelWithCurrentOptionsIfPossible() {
+ guard !isFinetuning else {
+ messageLog += "Finetuning in progress; runtime option will apply after the current run completes.\n"
+ return
+ }
+ guard let currentModelURL else { return }
+ do {
+ try loadModel(modelUrl: currentModelURL)
+ } catch {
+ messageLog += "Failed to reload model with new options: \(error.localizedDescription).\n"
+ }
+ }
+
+ private func applySamplerUpdate() {
+ guard let context = llamaContext else { return }
+ let options = runtimeOptions()
+ Task {
+ await context.updateSampler(options: options)
+ }
+ }
+
+ func selectDownloadedModel(_ model: Model) {
+ let fileURL = modelFileURL(for: model.filename)
+ guard FileManager.default.fileExists(atPath: fileURL.path) else {
+ messageLog += "Model \(model.filename) no longer exists on disk.\n"
+ return
+ }
+
+ do {
+ try loadModel(modelUrl: fileURL)
+ updateSelectedModel(filename: model.filename, name: model.name, announce: true)
+ } catch {
+ messageLog += "Failed to load model \(model.filename): \(error.localizedDescription).\n"
+ }
+ }
+
+ func registerDownloadedModel(_ model: Model) {
+ var downloaded = model
+ downloaded.status = "downloaded"
+
+ if let index = downloadedModels.firstIndex(where: { $0.filename == downloaded.filename }) {
+ downloadedModels[index] = downloaded
+ } else {
+ downloadedModels.append(downloaded)
+ }
+
+ undownloadedModels.removeAll { $0.filename == downloaded.filename }
+ downloadedModels.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ messageLog += "Downloaded model \(downloaded.name) -> \(downloaded.filename).\n"
+ }
+
+ func deleteModel(_ model: Model) {
+ let fileURL = modelFileURL(for: model.filename)
+ if FileManager.default.fileExists(atPath: fileURL.path) {
+ do {
+ try FileManager.default.removeItem(at: fileURL)
+ messageLog += "Deleted model \(model.filename).\n"
+ } catch {
+ messageLog += "Failed to delete model \(model.filename): \(error.localizedDescription).\n"
+ }
+ }
+
+ downloadedModels.removeAll { $0.filename == model.filename }
+
+ if !model.url.isEmpty && !undownloadedModels.contains(where: { $0.filename == model.filename }) {
+ var restorable = model
+ restorable.status = "download"
+ undownloadedModels.append(restorable)
+ undownloadedModels.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ }
+
+ if selectedModelFilename == model.filename {
+ updateSelectedModel(filename: nil, name: nil, announce: false)
+ }
+ }
+
+ func deleteDownloadedModels(at offsets: IndexSet) {
+ let targets = offsets.compactMap { index in
+ downloadedModels.indices.contains(index) ? downloadedModels[index] : nil
+ }
+
+ for model in targets {
+ deleteModel(model)
+ }
+ }
+
+ private func loadDatasetsFromDisk() {
+ downloadedDatasets.removeAll { !isDatasetFilename($0.filename) || isModelFilename($0.filename) }
+
+ do {
+ let documentsURL = getDocumentsDirectory()
+ let datasetURLs = try FileManager.default.contentsOfDirectory(
+ at: documentsURL,
+ includingPropertiesForKeys: nil,
+ options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants]
+ )
+
+ for datasetURL in datasetURLs where !datasetURL.hasDirectoryPath {
+ if !isDatasetFile(datasetURL) || isModelFilename(datasetURL.lastPathComponent) {
+ continue
+ }
+ let rawName = datasetURL.deletingPathExtension().lastPathComponent
+ let displayName = (rawName.removingPercentEncoding ?? rawName)
+ .replacingOccurrences(of: "_", with: " ")
+ .replacingOccurrences(of: "-", with: " ")
+ let dataset = Dataset(
+ name: displayName.isEmpty ? datasetURL.lastPathComponent : displayName,
+ url: "",
+ filename: datasetURL.lastPathComponent,
+ status: "downloaded"
+ )
+
+ if !downloadedDatasets.contains(where: { $0.filename == dataset.filename }) {
+ downloadedDatasets.append(dataset)
+ }
+ }
+ downloadedDatasets.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ } catch {
+ print("Error loading datasets from disk: \(error)")
+ }
+ }
+
+ private func loadDefaultDatasets() {
+ for dataset in defaultDatasets {
+ let fileURL = datasetFileURL(for: dataset.filename)
+ if FileManager.default.fileExists(atPath: fileURL.path) {
+ if !downloadedDatasets.contains(where: { $0.filename == dataset.filename }) {
+ var downloaded = dataset
+ downloaded.status = "downloaded"
+ downloadedDatasets.append(downloaded)
+ }
+ } else if !availableDatasets.contains(where: { $0.filename == dataset.filename }) {
+ availableDatasets.append(dataset)
+ }
+ }
+
+ downloadedDatasets.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ availableDatasets.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ }
+
+ private func ensureSelectedDatasetIsValid() {
+ if let selectedFilename = selectedDatasetFilename {
+ let selectedURL = datasetFileURL(for: selectedFilename)
+ if isDatasetFilename(selectedFilename)
+ && !isModelFilename(selectedFilename)
+ && FileManager.default.fileExists(atPath: selectedURL.path) {
+ return
+ }
+ }
+
+ if let firstDataset = downloadedDatasets.first {
+ updateSelectedDataset(filename: firstDataset.filename, name: firstDataset.name, announce: false)
+ } else {
+ updateSelectedDataset(filename: nil, name: nil, announce: false)
}
}
@@ -63,32 +468,139 @@ class LlamaState: ObservableObject {
let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
return paths[0]
}
+
+ func datasetFileURL(for filename: String) -> URL {
+ getDocumentsDirectory().appendingPathComponent(filename)
+ }
+
+ func modelFileURL(for filename: String) -> URL {
+ getDocumentsDirectory().appendingPathComponent(filename)
+ }
+
+ private func isDatasetFilename(_ filename: String) -> Bool {
+ let ext = URL(fileURLWithPath: filename).pathExtension.lowercased()
+ if ext.isEmpty {
+ return true
+ }
+ return !disallowedDatasetExtensions.contains(ext)
+ }
+
+ private func isDatasetFile(_ url: URL) -> Bool {
+ isDatasetFilename(url.lastPathComponent)
+ }
+
+ private func updateSelectedDataset(filename: String?, name: String?, announce: Bool) {
+ if let filename {
+ selectedDatasetFilename = filename
+ UserDefaults.standard.set(filename, forKey: datasetSelectionKey)
+ if announce {
+ let displayName = name ?? filename
+ messageLog += "Using dataset \(displayName) for finetuning.\n"
+ }
+ } else {
+ selectedDatasetFilename = nil
+ UserDefaults.standard.removeObject(forKey: datasetSelectionKey)
+ }
+ }
+
+ func selectDataset(_ dataset: Dataset) {
+ updateSelectedDataset(filename: dataset.filename, name: dataset.name, announce: true)
+ }
+
+ func registerDownloadedDataset(_ dataset: Dataset) {
+ guard isDatasetFilename(dataset.filename), !isModelFilename(dataset.filename) else {
+ let fileURL = datasetFileURL(for: dataset.filename)
+ if FileManager.default.fileExists(atPath: fileURL.path) {
+ do {
+ try FileManager.default.removeItem(at: fileURL)
+ messageLog += "Removed non-dataset file \(dataset.filename) from datasets folder.\n"
+ } catch {
+ messageLog += "Failed to remove non-dataset file \(dataset.filename): \(error.localizedDescription).\n"
+ }
+ }
+ messageLog += "Ignoring \(dataset.filename) because it is not a dataset file.\n"
+ return
+ }
+ var downloaded = dataset
+ downloaded.status = "downloaded"
+
+ if let index = downloadedDatasets.firstIndex(where: { $0.filename == downloaded.filename }) {
+ downloadedDatasets[index] = downloaded
+ } else {
+ downloadedDatasets.append(downloaded)
+ }
+
+ availableDatasets.removeAll { $0.filename == downloaded.filename }
+ downloadedDatasets.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ messageLog += "Downloaded dataset \(downloaded.name) -> \(downloaded.filename).\n"
+ updateSelectedDataset(filename: downloaded.filename, name: downloaded.name, announce: true)
+ }
+
+ private func isModelFilename(_ filename: String) -> Bool {
+ let ext = URL(fileURLWithPath: filename).pathExtension.lowercased()
+ if allowedModelExtensions.contains(ext) {
+ return true
+ }
+ return downloadedModels.contains { $0.filename == filename }
+ }
+
+ func deleteDataset(_ dataset: Dataset) {
+ let fileURL = datasetFileURL(for: dataset.filename)
+ if FileManager.default.fileExists(atPath: fileURL.path) {
+ do {
+ try FileManager.default.removeItem(at: fileURL)
+ messageLog += "Deleted dataset \(dataset.filename).\n"
+ } catch {
+ messageLog += "Failed to delete dataset \(dataset.filename): \(error.localizedDescription).\n"
+ }
+ }
+
+ downloadedDatasets.removeAll { $0.filename == dataset.filename }
+
+ if !dataset.url.isEmpty && !availableDatasets.contains(where: { $0.filename == dataset.filename }) {
+ var restorable = dataset
+ restorable.status = "download"
+ availableDatasets.append(restorable)
+ availableDatasets.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
+ }
+
+ if selectedDatasetFilename == dataset.filename {
+ updateSelectedDataset(filename: nil, name: nil, announce: false)
+ ensureSelectedDatasetIsValid()
+ }
+ }
+
+ func deleteDownloadedDatasets(at offsets: IndexSet) {
+ let targets = offsets.compactMap { index in
+ downloadedDatasets.indices.contains(index) ? downloadedDatasets[index] : nil
+ }
+
+ for dataset in targets {
+ deleteDataset(dataset)
+ }
+ }
private let defaultModels: [Model] = [
- Model(name: "TinyLlama-1.1B (Q4_0, 0.6 GiB)",url: "https://huggingface.co/TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF/resolve/main/tinyllama-1.1b-1t-openorca.Q4_0.gguf?download=true",filename: "tinyllama-1.1b-1t-openorca.Q4_0.gguf", status: "download"),
+ Model(name: "TinyLlama-1.1B (Q4_0, 0.6 GiB)", url: "https://huggingface.co/TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF/resolve/main/tinyllama-1.1b-1t-openorca.Q4_0.gguf?download=true", filename: "tinyllama-1.1b-1t-openorca.Q4_0.gguf", status: "download"),
Model(
name: "TinyLlama-1.1B Chat (Q8_0, 1.1 GiB)",
url: "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q8_0.gguf?download=true",
filename: "tinyllama-1.1b-chat-v1.0.Q8_0.gguf", status: "download"
),
-
Model(
name: "TinyLlama-1.1B (F16, 2.2 GiB)",
url: "https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf?download=true",
filename: "tinyllama-1.1b-f16.gguf", status: "download"
),
-
Model(
name: "Phi-2.7B (Q4_0, 1.6 GiB)",
url: "https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf?download=true",
filename: "phi-2-q4_0.gguf", status: "download"
),
-
Model(
name: "Phi-2.7B (Q8_0, 2.8 GiB)",
url: "https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q8_0.gguf?download=true",
filename: "phi-2-q8_0.gguf", status: "download"
),
-
Model(
name: "Mistral-7B-v0.1 (Q4_0, 3.8 GiB)",
url: "https://huggingface.co/TheBloke/Mistral-7B-v0.1-GGUF/resolve/main/mistral-7b-v0.1.Q4_0.gguf?download=true",
@@ -98,24 +610,133 @@ class LlamaState: ObservableObject {
name: "OpenHermes-2.5-Mistral-7B (Q3_K_M, 3.52 GiB)",
url: "https://huggingface.co/TheBloke/OpenHermes-2.5-Mistral-7B-GGUF/resolve/main/openhermes-2.5-mistral-7b.Q3_K_M.gguf?download=true",
filename: "openhermes-2.5-mistral-7b.Q3_K_M.gguf", status: "download"
+ ),
+ // Qwen3 0.6B family
+ Model(
+ name: "Qwen3-0.6B (Q4_0)",
+ url: "https://huggingface.co/medel/Qwen3-0.6B-q4_0.gguf/resolve/main/Qwen3-0.6B-q4_0.gguf",
+ filename: "Qwen3-0.6B-q4_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-0.6B (Q8_0)",
+ url: "https://huggingface.co/prithivMLmods/Qwen3-0.6B-GGUF/resolve/main/Qwen3_0.6B.Q8_0.gguf",
+ filename: "Qwen3_0.6B.Q8_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-0.6B (F16)",
+ url: "https://huggingface.co/medel/models-sink/resolve/main/Qwen3-0.6B-f16.gguf",
+ filename: "Qwen3-0.6B-f16.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-0.6B (F32)",
+ url: "https://huggingface.co/medel/models-sink/resolve/main/Qwen3-0.6B-f32.gguf",
+ filename: "Qwen3-0.6B-f32.gguf", status: "download"
+ ),
+ // Qwen3 1.7B family
+ Model(
+ name: "Qwen3-1.7B (Q4_0)",
+ url: "https://huggingface.co/medel/models-sink/resolve/main/Qwen3-1.7B-Q4_0.gguf",
+ filename: "Qwen3-1.7B-Q4_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-1.7B (Q8_0)",
+ url: "https://huggingface.co/Qwen/Qwen3-1.7B-GGUF/resolve/main/Qwen3-1.7B-Q8_0.gguf",
+ filename: "Qwen3-1.7B-Q8_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-1.7B (F16)",
+ url: "https://huggingface.co/medel/models-sink/resolve/main/Qwen3-1.7B-f16.gguf",
+ filename: "Qwen3-1.7B-f16.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-1.7B (F32)",
+ url: "https://huggingface.co/medel/models-sink/resolve/main/Qwen3-1.7B-f32.gguf",
+ filename: "Qwen3-1.7B-f32.gguf", status: "download"
+ ),
+ // Qwen3 4B family
+ Model(
+ name: "Qwen3-4B (Q4_0)",
+ url: "https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/main/Qwen3-4B-Q4_0.gguf",
+ filename: "Qwen3-4B-Q4_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-4B (Q8_0)",
+ url: "https://huggingface.co/unsloth/Qwen3-4B-GGUF/resolve/main/Qwen3-4B-Q8_0.gguf",
+ filename: "Qwen3-4B-Q8_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-4B (F16)",
+ url: "https://huggingface.co/medel/models-sink/resolve/main/Qwen3-4B-f16.gguf",
+ filename: "Qwen3-4B-f16.gguf", status: "download"
+ ),
+ Model(
+ name: "Qwen3-4B (F32)",
+ url: "https://huggingface.co/medel/models-sink/resolve/main/Qwen3-4B-f32.gguf",
+ filename: "Qwen3-4B-f32.gguf", status: "download"
+ ),
+ // Gemma 3 1B family
+ Model(
+ name: "Gemma3-1B Instruct (Q4_0)",
+ url: "https://huggingface.co/unsloth/gemma-3-1b-it-GGUF/resolve/main/gemma-3-1b-it-Q4_0.gguf",
+ filename: "gemma-3-1b-it-Q4_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Gemma3-1B Instruct (Q8_0)",
+ url: "https://huggingface.co/unsloth/gemma-3-1b-it-GGUF/resolve/main/gemma-3-1b-it-Q8_0.gguf",
+ filename: "gemma-3-1b-it-Q8_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Gemma3-1B Instruct (F16)",
+ url: "https://huggingface.co/gguf-org/gemma-3-1b-it-gguf/resolve/main/gemma-3-1b-it-f16.gguf",
+ filename: "gemma-3-1b-it-f16.gguf", status: "download"
+ ),
+ Model(
+ name: "Gemma3-1B Instruct (F32)",
+ url: "https://huggingface.co/gguf-org/gemma-3-1b-it-gguf/resolve/main/gemma-3-1b-it-f32.gguf",
+ filename: "gemma-3-1b-it-f32.gguf", status: "download"
+ ),
+ // Gemma 3 4B family
+ Model(
+ name: "Gemma3-4B Instruct (Q4_0)",
+ url: "https://huggingface.co/unsloth/gemma-3-4b-it-GGUF/resolve/main/gemma-3-4b-it-Q4_0.gguf",
+ filename: "gemma-3-4b-it-Q4_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Gemma3-4B Instruct (Q8_0)",
+ url: "https://huggingface.co/ggml-org/gemma-3-4b-it-GGUF/resolve/main/gemma-3-4b-it-Q8_0.gguf",
+ filename: "gemma-3-4b-it-Q8_0.gguf", status: "download"
+ ),
+ Model(
+ name: "Gemma3-4B Instruct (F16)",
+ url: "https://huggingface.co/ggml-org/gemma-3-4b-it-GGUF/resolve/main/gemma-3-4b-it-f16.gguf",
+ filename: "gemma-3-4b-it-f16.gguf", status: "download"
+ ),
+ Model(
+ name: "Gemma3-4B Instruct (F32)",
+ url: "https://huggingface.co/ggml-org/gemma-3-4b-it-GGUF/resolve/main/gemma-3-4b-it-f32.gguf",
+ filename: "gemma-3-4b-it-f32.gguf", status: "download"
)
]
func loadModel(modelUrl: URL?) throws {
- if let modelUrl {
- messageLog += "Loading model...\n"
- llamaContext = try LlamaContext.create_context(path: modelUrl.path())
- messageLog += "Loaded model \(modelUrl.lastPathComponent)\n"
-
- // Assuming that the model is successfully loaded, update the downloaded models
- updateDownloadedModels(modelName: modelUrl.lastPathComponent, status: "downloaded")
- } else {
+ guard let modelUrl else {
messageLog += "Load a model from the list below\n"
+ return
}
+
+ messageLog += "Loading model...\n"
+ llamaContext = try LlamaContext.create_context(path: modelUrl.path(), options: runtimeOptions())
+ messageLog += "Loaded model \(modelUrl.lastPathComponent)\n"
+ currentModelURL = modelUrl
+
+ let filename = modelUrl.lastPathComponent
+ updateDownloadedModels(filename: filename)
+ let displayName = modelUrl.deletingPathExtension().lastPathComponent
+ updateSelectedModel(filename: filename, name: displayName, announce: false)
}
- private func updateDownloadedModels(modelName: String, status: String) {
- undownloadedModels.removeAll { $0.name == modelName }
+ private func updateDownloadedModels(filename: String) {
+ undownloadedModels.removeAll { $0.filename == filename }
}
@@ -124,6 +745,12 @@ class LlamaState: ObservableObject {
return
}
+ await llamaContext.updateSampler(options: runtimeOptions())
+
+ if optionSingleTurn {
+ messageLog = ""
+ }
+
let t_start = DispatchTime.now().uptimeNanoseconds
await llamaContext.completion_init(text: text)
let t_heat_end = DispatchTime.now().uptimeNanoseconds
@@ -185,6 +812,130 @@ class LlamaState: ObservableObject {
messageLog += "\n"
}
+ func finetune() async {
+ if isFinetuning {
+ messageLog += "A finetuning run is already in progress.\n"
+ return
+ }
+
+ guard let modelURL = currentModelURL else {
+ messageLog += "Load a model before starting finetuning.\n"
+ return
+ }
+
+ guard let datasetURL = locateFinetuneDataset() else {
+ messageLog += "Unable to locate a finetuning dataset. Download one from Settings first.\n"
+ return
+ }
+
+ let outputURL = getDocumentsDirectory().appendingPathComponent("finetuned-\(modelURL.deletingPathExtension().lastPathComponent).gguf")
+ let outputFilename = outputURL.lastPathComponent
+
+ let ngl = optionNGpuLayers >= 0 ? Int32(optionNGpuLayers) : -1
+ let options = llama_swift_finetune_options(
+ n_ctx: Int32(optionContextLength),
+ n_threads: Int32(max(1, ProcessInfo.processInfo.processorCount - 1)),
+ n_batch: 256,
+ n_ubatch: 256,
+ epochs: 2,
+ lora_rank: 8,
+ lora_alpha: 16,
+ learning_rate: 1e-5,
+ val_split: 0.05,
+ target_modules: 0,
+ seed: Int32(optionSeed),
+ flash_attn: optionFlashAttention,
+ n_gpu_layers: ngl,
+ checkpoint_save_steps: 0,
+ checkpoint_save_dir: nil,
+ resume_from_checkpoint: nil,
+ auto_resume: false
+ )
+
+ isFinetuning = true
+ messageLog += "Starting on-device LoRA finetuning for \(modelURL.lastPathComponent).\n"
+
+ let modelPath = modelURL.path
+ let datasetPath = datasetURL.path
+ let outputPath = outputURL.path
+ let statePointer = Unmanaged.passUnretained(self).toOpaque()
+ let callback = LlamaState.finetuneLogCallback
+
+ Task.detached(priority: .userInitiated) {
+ var opts = options
+ let result: llama_swift_finetune_error = modelPath.withCString { modelC in
+ datasetPath.withCString { datasetC in
+ outputPath.withCString { outputC in
+ llama_swift_run_lora_finetune(modelC, datasetC, outputC, &opts, callback, statePointer)
+ }
+ }
+ }
+
+ Task { @MainActor in
+ let state = Unmanaged.fromOpaque(statePointer).takeUnretainedValue()
+ state.isFinetuning = false
+ if result == LLAMA_SWIFT_FINETUNE_OK {
+ state.messageLog += "Finetuning finished successfully. Adapter saved as \(outputFilename).\n"
+ } else {
+ state.messageLog += "Finetuning failed: \(state.describeFinetuneError(result)).\n"
+ }
+ }
+ }
+ }
+
+ private func locateFinetuneDataset() -> URL? {
+ let fileManager = FileManager.default
+ let environment = ProcessInfo.processInfo.environment
+
+ if let overridePath = environment["LLAMA_FINETUNE_DATASET"], !overridePath.isEmpty {
+ let overrideURL = URL(fileURLWithPath: overridePath)
+ if fileManager.fileExists(atPath: overrideURL.path) {
+ return overrideURL
+ }
+ }
+
+ if let selectedFilename = selectedDatasetFilename {
+ let selectedURL = datasetFileURL(for: selectedFilename)
+ if fileManager.fileExists(atPath: selectedURL.path) {
+ return selectedURL
+ }
+ }
+
+ for dataset in downloadedDatasets {
+ let candidate = datasetFileURL(for: dataset.filename)
+ if fileManager.fileExists(atPath: candidate.path) {
+ updateSelectedDataset(filename: dataset.filename, name: dataset.name, announce: false)
+ return candidate
+ }
+ }
+
+ let workingDirectoryCandidate = URL(fileURLWithPath: FileManager.default.currentDirectoryPath).appendingPathComponent(finetuneDatasetFilename)
+ if fileManager.fileExists(atPath: workingDirectoryCandidate.path) {
+ return workingDirectoryCandidate
+ }
+
+ return nil
+ }
+
+ private func describeFinetuneError(_ code: llama_swift_finetune_error) -> String {
+ switch code {
+ case LLAMA_SWIFT_FINETUNE_ERROR_INVALID_ARGUMENT:
+ return "invalid configuration"
+ case LLAMA_SWIFT_FINETUNE_ERROR_MODEL_LOAD:
+ return "failed to load model"
+ case LLAMA_SWIFT_FINETUNE_ERROR_CONTEXT_CREATE:
+ return "failed to create training context"
+ case LLAMA_SWIFT_FINETUNE_ERROR_DATASET:
+ return "dataset preparation failed"
+ case LLAMA_SWIFT_FINETUNE_ERROR_TRAINING_INIT:
+ return "training initialization failed"
+ case LLAMA_SWIFT_FINETUNE_ERROR_SAVE:
+ return "failed to save adapter"
+ default:
+ return "unexpected error (\(code.rawValue))"
+ }
+ }
+
func clear() async {
guard let llamaContext else {
return
@@ -193,4 +944,5 @@ class LlamaState: ObservableObject {
await llamaContext.clear()
messageLog = ""
}
+
}
diff --git a/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift b/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift
index 1c3cd9d2efc7..497095055daf 100644
--- a/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift
+++ b/examples/llama.swiftui/llama.swiftui/UI/ContentView.swift
@@ -32,6 +32,11 @@ struct ContentView: View {
bench()
}
+ Button(llamaState.isFinetuning ? "Finetuning..." : "Finetune") {
+ finetune()
+ }
+ .disabled(llamaState.isFinetuning)
+
Button("Clear") {
clear()
}
@@ -43,14 +48,14 @@ struct ContentView: View {
.buttonStyle(.bordered)
.padding()
- NavigationLink(destination: DrawerView(llamaState: llamaState)) {
- Text("View Models")
+ NavigationLink(destination: SettingsView(llamaState: llamaState)) {
+ Text("Settings")
}
.padding()
}
.padding()
- .navigationBarTitle("Model Settings", displayMode: .inline)
+ .navigationBarTitle("Settings", displayMode: .inline)
}
}
@@ -68,56 +73,184 @@ struct ContentView: View {
}
}
+ func finetune() {
+ Task {
+ await llamaState.finetune()
+ }
+ }
+
func clear() {
Task {
await llamaState.clear()
}
}
- struct DrawerView: View {
+ struct SettingsView: View {
@ObservedObject var llamaState: LlamaState
@State private var showingHelp = false
- func delete(at offsets: IndexSet) {
- offsets.forEach { offset in
- let model = llamaState.downloadedModels[offset]
- let fileURL = getDocumentsDirectory().appendingPathComponent(model.filename)
- do {
- try FileManager.default.removeItem(at: fileURL)
- } catch {
- print("Error deleting file: \(error)")
+ var body: some View {
+ List {
+ Section(header: Text("Dataset for Finetuning")) {
+ if let selectedFilename = llamaState.selectedDatasetFilename,
+ let dataset = llamaState.downloadedDatasets.first(where: { $0.filename == selectedFilename }) {
+ VStack(alignment: .leading, spacing: 4) {
+ Text(dataset.name)
+ .font(.body)
+ Text(dataset.filename)
+ .font(.caption)
+ .foregroundColor(.secondary)
+ }
+ } else {
+ Text("No dataset selected. Download or choose one below.")
+ .font(.footnote)
+ .foregroundColor(.secondary)
+ }
}
- }
- // Remove models from downloadedModels array
- llamaState.downloadedModels.remove(atOffsets: offsets)
- }
+ if !llamaState.downloadedDatasets.isEmpty {
+ Section(header: Text("Downloaded Datasets")) {
+ ForEach(llamaState.downloadedDatasets) { dataset in
+ DatasetDownloadedRow(llamaState: llamaState, dataset: dataset)
+ }
+ .onDelete { offsets in
+ llamaState.deleteDownloadedDatasets(at: offsets)
+ }
+ }
+ }
- func getDocumentsDirectory() -> URL {
- let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)
- return paths[0]
- }
- var body: some View {
- List {
- Section(header: Text("Download Models From Hugging Face")) {
- HStack {
- InputButton(llamaState: llamaState)
+ if !llamaState.availableDatasets.isEmpty {
+ Section(header: Text("Available Datasets")) {
+ ForEach(llamaState.availableDatasets) { dataset in
+ DatasetAvailableRow(llamaState: llamaState, dataset: dataset)
+ }
}
}
- Section(header: Text("Downloaded Models")) {
- ForEach(llamaState.downloadedModels) { model in
- DownloadButton(llamaState: llamaState, modelName: model.name, modelUrl: model.url, filename: model.filename)
+
+ Section(header: Text("Download Dataset From URL")) {
+ DatasetURLInputRow(llamaState: llamaState)
+ }
+ Section(header: Text("Model for Inference")) {
+ if let selectedFilename = llamaState.selectedModelFilename,
+ let model = llamaState.downloadedModels.first(where: { $0.filename == selectedFilename }) {
+ VStack(alignment: .leading, spacing: 4) {
+ Text(model.name)
+ .font(.body)
+ Text(model.filename)
+ .font(.caption)
+ .foregroundColor(.secondary)
+ }
+ } else if let selectedFilename = llamaState.selectedModelFilename {
+ VStack(alignment: .leading, spacing: 4) {
+ Text(selectedFilename)
+ .font(.body)
+ Text("Bundle resource")
+ .font(.caption)
+ .foregroundColor(.secondary)
+ }
+ } else {
+ Text("No model loaded. Download or choose one below.")
+ .font(.footnote)
+ .foregroundColor(.secondary)
}
- .onDelete(perform: delete)
}
- Section(header: Text("Default Models")) {
- ForEach(llamaState.undownloadedModels) { model in
- DownloadButton(llamaState: llamaState, modelName: model.name, modelUrl: model.url, filename: model.filename)
+
+ if !llamaState.downloadedModels.isEmpty {
+ Section(header: Text("Downloaded Models")) {
+ ForEach(llamaState.downloadedModels) { model in
+ ModelDownloadedRow(llamaState: llamaState, model: model)
+ }
+ .onDelete { offsets in
+ llamaState.deleteDownloadedModels(at: offsets)
+ }
+ }
+ }
+
+ if !llamaState.undownloadedModels.isEmpty {
+ Section(header: Text("Available Models")) {
+ ForEach(llamaState.undownloadedModels) { model in
+ ModelAvailableRow(llamaState: llamaState, model: model)
+ }
+ }
+ }
+
+ Section(header: Text("Download Model From URL")) {
+ ModelURLInputRow(llamaState: llamaState)
+ }
+
+ Section(header: Text("Finetuning & Runtime Options")) {
+ Stepper(value: $llamaState.optionNGpuLayers, in: -1...256, step: 1) {
+ HStack {
+ Text("n-gpu-layers (ngl)")
+ Spacer()
+ if llamaState.optionNGpuLayers >= 0 {
+ Text("\(llamaState.optionNGpuLayers)")
+ .foregroundColor(.secondary)
+ } else {
+ Text("Auto")
+ .foregroundColor(.secondary)
+ }
+ }
+ }
+
+ Stepper(value: $llamaState.optionContextLength, in: 32...8192, step: 64) {
+ HStack {
+ Text("Context size (c)")
+ Spacer()
+ Text("\(llamaState.optionContextLength)")
+ .foregroundColor(.secondary)
+ }
+ }
+
+ Stepper(value: $llamaState.optionSeed, in: 0...1_000_000, step: 1) {
+ HStack {
+ Text("Seed (s)")
+ Spacer()
+ Text("\(llamaState.optionSeed)")
+ .foregroundColor(.secondary)
+ }
+ }
+
+ VStack(alignment: .leading, spacing: 8) {
+ HStack {
+ Text("Temperature (temp)")
+ Spacer()
+ Text(String(format: "%.2f", llamaState.optionTemperature))
+ .foregroundColor(.secondary)
+ }
+ Slider(value: $llamaState.optionTemperature, in: 0...0)
+ }
+
+ VStack(alignment: .leading, spacing: 8) {
+ HStack {
+ Text("Top-p")
+ Spacer()
+ Text(String(format: "%.2f", llamaState.optionTopP))
+ .foregroundColor(.secondary)
+ }
+ Slider(value: $llamaState.optionTopP, in: 0.01...1.0, step: 0.01)
+ }
+
+ Stepper(value: $llamaState.optionTopK, in: 1...500, step: 1) {
+ HStack {
+ Text("Top-k")
+ Spacer()
+ Text("\(llamaState.optionTopK)")
+ .foregroundColor(.secondary)
+ }
+ }
+
+ Toggle(isOn: $llamaState.optionFlashAttention) {
+ Text("Flash Attention")
+ }
+
+ Toggle(isOn: $llamaState.optionSingleTurn) {
+ Text("Single Turn (st)")
}
}
}
.listStyle(GroupedListStyle())
- .navigationBarTitle("Model Settings", displayMode: .inline).toolbar {
+ .navigationBarTitle("Settings", displayMode: .inline).toolbar {
ToolbarItem(placement: .navigationBarTrailing) {
Button("Help") {
showingHelp = true
@@ -149,6 +282,486 @@ struct ContentView: View {
}
}
+struct DatasetDownloadedRow: View {
+ @ObservedObject var llamaState: LlamaState
+ let dataset: Dataset
+
+ private var isSelected: Bool {
+ llamaState.selectedDatasetFilename == dataset.filename
+ }
+
+ var body: some View {
+ Button {
+ llamaState.selectDataset(dataset)
+ } label: {
+ HStack(alignment: .center, spacing: 12) {
+ VStack(alignment: .leading, spacing: 4) {
+ Text(dataset.name)
+ .foregroundColor(.primary)
+ Text(dataset.filename)
+ .font(.caption)
+ .foregroundColor(.secondary)
+ }
+ Spacer()
+ if isSelected {
+ Image(systemName: "checkmark.circle.fill")
+ .foregroundColor(.accentColor)
+ }
+ }
+ .padding(.vertical, 4)
+ }
+ .buttonStyle(.plain)
+ }
+}
+
+struct DatasetAvailableRow: View {
+ @ObservedObject var llamaState: LlamaState
+ let dataset: Dataset
+ @State private var status: Status = .idle
+
+ private enum Status: Equatable {
+ case idle
+ case downloading
+ case failed(String)
+ }
+
+ var body: some View {
+ VStack(alignment: .leading, spacing: 8) {
+ Text(dataset.name)
+ .font(.body)
+ Text(dataset.url)
+ .font(.caption)
+ .foregroundColor(.secondary)
+ .lineLimit(2)
+
+ switch status {
+ case .idle:
+ Button("Download") {
+ startDownload()
+ }
+ .buttonStyle(.borderedProminent)
+ case .downloading:
+ HStack(spacing: 8) {
+ ProgressView()
+ Text("Downloading…")
+ .font(.footnote)
+ .foregroundColor(.secondary)
+ }
+ case .failed(let message):
+ VStack(alignment: .leading, spacing: 4) {
+ Text("Download failed: \(message)")
+ .font(.caption)
+ .foregroundColor(.red)
+ Button("Retry") {
+ status = .idle
+ startDownload()
+ }
+ }
+ }
+ }
+ .padding(.vertical, 4)
+ }
+
+ private func startDownload() {
+ guard status != .downloading else { return }
+ guard let downloadURL = URL(string: dataset.url) else {
+ status = .failed("Invalid URL")
+ return
+ }
+
+ status = .downloading
+
+ Task {
+ do {
+ let (temporaryURL, response) = try await URLSession.shared.download(from: downloadURL)
+ guard let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) else {
+ await MainActor.run {
+ status = .failed("Server error")
+ }
+ return
+ }
+
+ let destination = await MainActor.run {
+ llamaState.datasetFileURL(for: dataset.filename)
+ }
+
+ let fileManager = FileManager.default
+ if fileManager.fileExists(atPath: destination.path) {
+ try fileManager.removeItem(at: destination)
+ }
+ try fileManager.moveItem(at: temporaryURL, to: destination)
+
+ await MainActor.run {
+ llamaState.registerDownloadedDataset(dataset)
+ status = .idle
+ }
+ } catch {
+ if Task.isCancelled { return }
+ await MainActor.run {
+ status = .failed(error.localizedDescription)
+ }
+ }
+ }
+ }
+}
+
+struct DatasetURLInputRow: View {
+ @ObservedObject var llamaState: LlamaState
+ @State private var urlText: String = ""
+ @State private var status: Status = .idle
+
+ private enum Status: Equatable {
+ case idle
+ case downloading
+ case failed(String)
+ case success
+ }
+
+ var body: some View {
+ VStack(alignment: .leading, spacing: 8) {
+ TextField("https://example.com/dataset.jsonl", text: $urlText)
+ .textFieldStyle(RoundedBorderTextFieldStyle())
+ .autocorrectionDisabled(true)
+ .textInputAutocapitalization(.never)
+
+ switch status {
+ case .failed(let message):
+ Text(message)
+ .font(.caption)
+ .foregroundColor(.red)
+ case .success:
+ Text("Download complete!")
+ .font(.caption)
+ .foregroundColor(.green)
+ default:
+ EmptyView()
+ }
+
+ Button(action: startDownload) {
+ if status == .downloading {
+ HStack {
+ ProgressView()
+ Text("Downloading…")
+ }
+ } else {
+ Text("Download Dataset")
+ }
+ }
+ .buttonStyle(.borderedProminent)
+ .disabled(status == .downloading || urlText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
+ }
+ .padding(.vertical, 4)
+ }
+
+ private func startDownload() {
+ guard status != .downloading else { return }
+ let trimmed = urlText.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard let downloadURL = URL(string: trimmed), !trimmed.isEmpty else {
+ status = .failed("Enter a valid dataset URL")
+ return
+ }
+
+ let filename = deriveFilename(from: downloadURL)
+ let displayName = deriveDisplayName(from: filename)
+ let dataset = Dataset(name: displayName, url: trimmed, filename: filename, status: "download")
+
+ status = .downloading
+
+ Task {
+ do {
+ let (temporaryURL, response) = try await URLSession.shared.download(from: downloadURL)
+ guard let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) else {
+ await MainActor.run {
+ status = .failed("Server error")
+ }
+ return
+ }
+
+ let destination = await MainActor.run {
+ llamaState.datasetFileURL(for: dataset.filename)
+ }
+
+ let fileManager = FileManager.default
+ if fileManager.fileExists(atPath: destination.path) {
+ try fileManager.removeItem(at: destination)
+ }
+ try fileManager.moveItem(at: temporaryURL, to: destination)
+
+ await MainActor.run {
+ llamaState.registerDownloadedDataset(dataset)
+ status = .success
+ urlText = ""
+ }
+ } catch {
+ if Task.isCancelled { return }
+ await MainActor.run {
+ status = .failed(error.localizedDescription)
+ }
+ }
+ }
+ }
+
+ private func deriveFilename(from url: URL) -> String {
+ let candidate = url.lastPathComponent
+ if candidate.isEmpty {
+ return "dataset-\(UUID().uuidString).txt"
+ }
+ return candidate
+ }
+
+ private func deriveDisplayName(from filename: String) -> String {
+ let decoded = filename.removingPercentEncoding ?? filename
+ let baseName = (decoded as NSString).deletingPathExtension
+ let cleaned = baseName.replacingOccurrences(of: "_", with: " ")
+ .replacingOccurrences(of: "-", with: " ")
+ let trimmed = cleaned.trimmingCharacters(in: .whitespacesAndNewlines)
+ if trimmed.isEmpty {
+ return "Custom Dataset"
+ }
+ return trimmed.capitalized
+ }
+}
+
+struct ModelDownloadedRow: View {
+ @ObservedObject var llamaState: LlamaState
+ let model: Model
+
+ private var isSelected: Bool {
+ llamaState.selectedModelFilename == model.filename
+ }
+
+ var body: some View {
+ Button {
+ llamaState.selectDownloadedModel(model)
+ } label: {
+ HStack(alignment: .center, spacing: 12) {
+ VStack(alignment: .leading, spacing: 4) {
+ Text(model.name)
+ .foregroundColor(.primary)
+ Text(model.filename)
+ .font(.caption)
+ .foregroundColor(.secondary)
+ }
+ Spacer()
+ if isSelected {
+ Image(systemName: "checkmark.circle.fill")
+ .foregroundColor(.accentColor)
+ }
+ }
+ .padding(.vertical, 4)
+ }
+ .buttonStyle(.plain)
+ }
+}
+
+struct ModelAvailableRow: View {
+ @ObservedObject var llamaState: LlamaState
+ let model: Model
+ @State private var status: Status = .idle
+
+ private enum Status: Equatable {
+ case idle
+ case downloading
+ case failed(String)
+ }
+
+ var body: some View {
+ VStack(alignment: .leading, spacing: 8) {
+ Text(model.name)
+ .font(.body)
+ Text(model.url)
+ .font(.caption)
+ .foregroundColor(.secondary)
+ .lineLimit(2)
+
+ switch status {
+ case .idle:
+ Button("Download") {
+ startDownload()
+ }
+ .buttonStyle(.borderedProminent)
+ case .downloading:
+ HStack(spacing: 8) {
+ ProgressView()
+ Text("Downloading…")
+ .font(.footnote)
+ .foregroundColor(.secondary)
+ }
+ case .failed(let message):
+ VStack(alignment: .leading, spacing: 4) {
+ Text("Download failed: \(message)")
+ .font(.caption)
+ .foregroundColor(.red)
+ Button("Retry") {
+ status = .idle
+ startDownload()
+ }
+ }
+ }
+ }
+ .padding(.vertical, 4)
+ }
+
+ private func startDownload() {
+ guard status != .downloading else { return }
+ guard let downloadURL = URL(string: model.url) else {
+ status = .failed("Invalid URL")
+ return
+ }
+
+ status = .downloading
+
+ Task {
+ do {
+ let (temporaryURL, response) = try await URLSession.shared.download(from: downloadURL)
+ guard let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) else {
+ await MainActor.run {
+ status = .failed("Server error")
+ }
+ return
+ }
+
+ let destination = await MainActor.run {
+ llamaState.modelFileURL(for: model.filename)
+ }
+
+ let fileManager = FileManager.default
+ if fileManager.fileExists(atPath: destination.path) {
+ try fileManager.removeItem(at: destination)
+ }
+ try fileManager.moveItem(at: temporaryURL, to: destination)
+
+ await MainActor.run {
+ llamaState.registerDownloadedModel(model)
+ status = .idle
+ }
+ } catch {
+ if Task.isCancelled { return }
+ await MainActor.run {
+ status = .failed(error.localizedDescription)
+ }
+ }
+ }
+ }
+}
+
+struct ModelURLInputRow: View {
+ @ObservedObject var llamaState: LlamaState
+ @State private var urlText: String = ""
+ @State private var status: Status = .idle
+
+ private enum Status: Equatable {
+ case idle
+ case downloading
+ case failed(String)
+ case success
+ }
+
+ var body: some View {
+ VStack(alignment: .leading, spacing: 8) {
+ TextField("https://example.com/model.gguf", text: $urlText)
+ .textFieldStyle(RoundedBorderTextFieldStyle())
+ .autocorrectionDisabled(true)
+ .textInputAutocapitalization(.never)
+
+ switch status {
+ case .failed(let message):
+ Text(message)
+ .font(.caption)
+ .foregroundColor(.red)
+ case .success:
+ Text("Download complete!")
+ .font(.caption)
+ .foregroundColor(.green)
+ default:
+ EmptyView()
+ }
+
+ Button(action: startDownload) {
+ if status == .downloading {
+ HStack {
+ ProgressView()
+ Text("Downloading…")
+ }
+ } else {
+ Text("Download Model")
+ }
+ }
+ .buttonStyle(.borderedProminent)
+ .disabled(status == .downloading || urlText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
+ }
+ .padding(.vertical, 4)
+ }
+
+ private func startDownload() {
+ guard status != .downloading else { return }
+ let trimmed = urlText.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard let downloadURL = URL(string: trimmed), !trimmed.isEmpty else {
+ status = .failed("Enter a valid model URL")
+ return
+ }
+
+ let filename = deriveFilename(from: downloadURL)
+ let displayName = deriveDisplayName(from: filename)
+ let model = Model(name: displayName, url: trimmed, filename: filename, status: "download")
+
+ status = .downloading
+
+ Task {
+ do {
+ let (temporaryURL, response) = try await URLSession.shared.download(from: downloadURL)
+ guard let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) else {
+ await MainActor.run {
+ status = .failed("Server error")
+ }
+ return
+ }
+
+ let destination = await MainActor.run {
+ llamaState.modelFileURL(for: model.filename)
+ }
+
+ let fileManager = FileManager.default
+ if fileManager.fileExists(atPath: destination.path) {
+ try fileManager.removeItem(at: destination)
+ }
+ try fileManager.moveItem(at: temporaryURL, to: destination)
+
+ await MainActor.run {
+ llamaState.registerDownloadedModel(model)
+ status = .success
+ urlText = ""
+ }
+ } catch {
+ if Task.isCancelled { return }
+ await MainActor.run {
+ status = .failed(error.localizedDescription)
+ }
+ }
+ }
+ }
+
+ private func deriveFilename(from url: URL) -> String {
+ let candidate = url.lastPathComponent
+ if candidate.isEmpty {
+ return "model-\(UUID().uuidString).gguf"
+ }
+ return candidate
+ }
+
+ private func deriveDisplayName(from filename: String) -> String {
+ let decoded = filename.removingPercentEncoding ?? filename
+ let baseName = (decoded as NSString).deletingPathExtension
+ let cleaned = baseName.replacingOccurrences(of: "_", with: " ")
+ .replacingOccurrences(of: "-", with: " ")
+ let trimmed = cleaned.trimmingCharacters(in: .whitespacesAndNewlines)
+ if trimmed.isEmpty {
+ return "Custom Model"
+ }
+ return trimmed.capitalized
+ }
+}
+
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
ContentView()
diff --git a/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift b/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift
index 4584d6eaa3d3..631ce50c5019 100644
--- a/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift
+++ b/examples/llama.swiftui/llama.swiftui/UI/DownloadButton.swift
@@ -1,3 +1,4 @@
+#if false
import SwiftUI
struct DownloadButton: View {
@@ -113,12 +114,13 @@ struct DownloadButton: View {
}
}
}
+#endif
-// #Preview {
-// DownloadButton(
-// llamaState: LlamaState(),
-// modelName: "TheBloke / TinyLlama-1.1B-1T-OpenOrca-GGUF (Q4_0)",
-// modelUrl: "https://huggingface.co/TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF/resolve/main/tinyllama-1.1b-1t-openorca.Q4_0.gguf?download=true",
-// filename: "tinyllama-1.1b-1t-openorca.Q4_0.gguf"
-// )
-// }
+import SwiftUI
+
+@available(*, deprecated, message: "Use ModelAvailableRow and ModelDownloadedRow instead.")
+struct DownloadButton: View {
+ var body: some View {
+ EmptyView()
+ }
+}
diff --git a/examples/llama.swiftui/llama.swiftui/UI/InputButton.swift b/examples/llama.swiftui/llama.swiftui/UI/InputButton.swift
index c5ffbad4ec33..60bf1d7b0fb9 100644
--- a/examples/llama.swiftui/llama.swiftui/UI/InputButton.swift
+++ b/examples/llama.swiftui/llama.swiftui/UI/InputButton.swift
@@ -1,3 +1,4 @@
+#if false
import SwiftUI
struct InputButton: View {
@@ -129,3 +130,13 @@ struct InputButton: View {
}
}
}
+#endif
+
+import SwiftUI
+
+@available(*, deprecated, message: "Use ModelURLInputRow instead.")
+struct InputButton: View {
+ var body: some View {
+ EmptyView()
+ }
+}
diff --git a/examples/simple-cmake-pkg/CMakeLists.txt b/examples/simple-cmake-pkg/CMakeLists.txt
index 128e38c8f2dc..06e2692a1d85 100644
--- a/examples/simple-cmake-pkg/CMakeLists.txt
+++ b/examples/simple-cmake-pkg/CMakeLists.txt
@@ -7,5 +7,5 @@ find_package(Llama REQUIRED)
add_executable(${TARGET} ${CMAKE_CURRENT_LIST_DIR}/../simple/simple.cpp)
install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE llama ggml::all ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE llama::llama ggml::all ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_17)
diff --git a/examples/simple/CMakeLists.txt b/examples/simple/CMakeLists.txt
index 104ecabfd723..5ada3fdd3de6 100644
--- a/examples/simple/CMakeLists.txt
+++ b/examples/simple/CMakeLists.txt
@@ -1,5 +1,5 @@
set(TARGET llama-simple)
add_executable(${TARGET} simple.cpp)
install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE llama llama-common-test ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_17)
diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp
index d09771d10457..0268f503fe2b 100644
--- a/examples/simple/simple.cpp
+++ b/examples/simple/simple.cpp
@@ -1,15 +1,20 @@
+#include "llama-cpp.h"
#include "llama.h"
#include
#include
#include
-#include
static void print_usage(int, char ** argv) {
printf("\nexample usage:\n");
printf("\n %s -m model.gguf [-n n_predict] [-ngl n_gpu_layers] [prompt]\n", argv[0]);
+ printf("\n Optional environment variables: LLAMA_EXAMPLE_MEMORY_BUFFER LLAMA_EXAMPLE_MEMORY_BUFFER_SPLIT");
printf("\n");
}
+#ifdef LLAMA_COMMON_TEST_HEADERS
+#include "load_into_memory.h"
+#endif
+
int main(int argc, char ** argv) {
// path to the model gguf file
std::string model_path;
@@ -83,12 +88,13 @@ int main(int argc, char ** argv) {
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = ngl;
+#ifdef LLAMA_COMMON_TEST_HEADERS
+ llama_model * model = memory_configuration_env_is_set() ?
+ load_model_from_memory_configuration(model_path.c_str(), model_params) :
+ llama_model_load_from_file(model_path.c_str(), model_params);
+#else
llama_model * model = llama_model_load_from_file(model_path.c_str(), model_params);
-
- if (model == NULL) {
- fprintf(stderr , "%s: error: unable to load model\n" , __func__);
- return 1;
- }
+#endif
const llama_vocab * vocab = llama_model_get_vocab(model);
// tokenize the prompt
diff --git a/examples/training/CMakeLists.txt b/examples/training/CMakeLists.txt
index 64afe6ddc647..c11e981dbf3b 100644
--- a/examples/training/CMakeLists.txt
+++ b/examples/training/CMakeLists.txt
@@ -3,3 +3,9 @@ add_executable(${TARGET} finetune.cpp)
install(TARGETS ${TARGET} RUNTIME)
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
target_compile_features(${TARGET} PRIVATE cxx_std_11)
+
+set(TARGET llama-finetune-lora)
+add_executable(${TARGET} finetune-lora.cpp)
+install(TARGETS ${TARGET} RUNTIME)
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(${TARGET} PRIVATE cxx_std_11)
diff --git a/examples/training/README.md b/examples/training/README.md
index df425279266e..3f46b50fd9c5 100644
--- a/examples/training/README.md
+++ b/examples/training/README.md
@@ -1,5 +1,6 @@
# llama.cpp/examples/training
+## finetune
This directory contains examples related to language model training using llama.cpp/GGML.
So far finetuning is technically functional (for FP32 models and limited hardware setups) but the code is very much WIP.
Finetuning of Stories 260K and LLaMA 3.2 1b seems to work with 24 GB of memory.
@@ -15,3 +16,266 @@ export model_name=llama_3.2-1b && export quantization=f32
```
The perplexity value of the finetuned model should be lower after training on the test set for 2 epochs.
+
+
+## finetune-lora
+
+LoRA (Low-Rank Adaptation) fine-tuning for efficient model training. This approach trains only a small set of additional parameters while keeping
+the base model frozen, making it memory-efficient.
+
+### Basic Usage
+
+```sh
+# Create new LoRA adapter with default settings (rank=8, alpha=16, attention modules)
+./build/bin/llama-finetune-lora -m model.gguf -f dataset.txt -ngl 999 -c 512 -b 512 -ub 512 -fa off
+
+# Custom LoRA parameters(creates new lora adapter and trains it from scratch)
+./build/bin/llama-finetune-lora -m model.gguf -f dataset.txt -ngl 999 -c 512 -b 512 -ub 512 \
+ --lora-rank 16 --lora-alpha 32 --lora-modules "attn_q,attn_k,attn_v,attn_o" -fa off
+
+# Fine-tune existing LoRA adapter
+./build/bin/llama-finetune-lora -m base_model.gguf -f dataset.txt --lora existing_adapter.gguf \
+ --output-adapter improved_adapter.gguf -ngl 999 -c 512 -b 512 -ub 512
+
+# Training with checkpointing
+./build/bin/llama-finetune-lora -m model.gguf -f dataset.txt -ngl 999 -c 512 -b 512 -ub 512 \
+ --checkpoint-save-steps 50 --checkpoint-save-dir "./lora_checkpoints"
+
+# Resume training from checkpoint
+./build/bin/llama-finetune-lora -m model.gguf -f dataset.txt -ngl 999 -c 512 -b 512 -ub 512 \
+ --resume-from "./lora_checkpoints/checkpoint_step_00000150/"
+ --output-adapter improved_adapter.gguf -ngl 999 -c 512 -b 512 -ub 512 -fa off
+
+# Supervised FineTuning with Assistant only loss
+./build/bin/llama-finetune-lora -m model.gguf -f dataset.jsonl -ngl 999 -c 512 -b 512 -ub 512 \
+ --lora-modules "attn_q,attn_k,attn_v,attn_o" --assistant-loss-only -fa off
+```
+
+### SFT(Instruction Fine Tuning) with Assistant Only Loss
+- Masks the system and user tokens and only computes loss on assistant tokens
+- Requires the dataset to be in json format just like huggingface with `role` and `content` for each role
+- Allows users to optionally pass a jinja chat template with `--chat-template chat-ml-template.jinja`
+
+### Parameters
+
+#### LoRA Configuration
+- `--lora-rank N` - LoRA rank (default: 8)
+ - Lower rank = smaller adapter, less capacity
+ - Higher rank = larger adapter, more capacity
+- `--lora-alpha N` - LoRA alpha scaling factor (default: 16.0)
+ - Controls adaptation strength
+ - Common rule: alpha = 2 × rank
+- `--lora-modules MODULES` - Target modules as comma-separated list
+ - Available: `attn_q`, `attn_k`, `attn_v`, `attn_o`, `ffn_gate`, `ffn_up`, `ffn_down`, `embed`, `output`, `all`
+ - Default: `attn_q,attn_k,attn_v,attn_o` (attention modules)
+- `--output-adapter PATH` - Output adapter filename (default: auto-generated)
+- `--lora-seed N` - Seed for reproducible LoRA weight initialization (default: 0 = non-deterministic)
+
+#### Checkpointing
+- `--checkpoint-save-steps N` - Save checkpoint every N training steps (default: 100)
+- `--checkpoint-save-dir PATH` - Directory for checkpoints (default: `./checkpoints`)
+- `--resume-from PATH` - Resume training from specific checkpoint directory
+- `--auto-resume` - Automatically resume from latest checkpoint in save directory
+
+#### Standard Parameters
+- `-m MODEL` - Base model file (.gguf)
+- `-f FILE` - Training dataset
+- `-ngl N` - GPU layers (use 999 for full GPU training)
+- `-c N` - Context length (512 recommended for mobile)
+- `--assistant-loss-only` - Trains only on assistant tokens
+- `--chat-template` - Jinja chat template for chat ML formatting to train on assistant tokens only
+- `--learning-rate` - AdamW learning rate (default: 1e-5)
+- `--weight-decay` - AdamW weight decay (default: 1e-2)
+- `--lr-scheduler` - Learning rate scheduler: constant, cosine, linear (default: constant)
+- `--lr-min` - Minimum LR for cosine/linear schedulers (default: 0)
+- `--warmup-ratio` - Fraction of total steps for LR warmup (default: 0.0)
+- `--warmup-steps` - Explicit warmup steps (overrides warmup-ratio)
+
+
+### Using Trained Adapters
+
+After training, you'll get a small adapter file. Use it with the original base model:
+
+```sh
+./build/bin/llama-cli -m base_model.gguf --lora trained_adapter.gguf -ngl 999
+```
+
+### Checkpointing
+
+The LoRA fine-tuning supports automatic checkpointing to save and resume training progress:
+- **Automatic saving**: Model and optimizer state saved every N training steps
+- **Complete state**: Includes LoRA weights, optimizer momentum, and training metadata
+- **Resume capability**: Continue training from exact step with full optimizer state
+- **Auto-resume**: Automatically find and resume from latest checkpoint
+
+#### Checkpoint Structure
+Each checkpoint directory contains:
+- `model.gguf` - LoRA adapter weights
+- `optimizer.gguf` - Optimizer state (momentum, variance, iteration)
+- `metadata.txt` - Training parameters and step information
+
+### Architecture Overview
+
+This section explains how LoRA fine-tuning is implemented in llama.cpp:
+
+**LoRA Adapter Management (`src/llama-lora-training.cpp`):**
+This file manages the complete lifecycle of LoRA adapters:
+
+1. **Adapter Creation (`llama_lora_create_adapter()`):**
+ - Iterates through all model tensors to find target modules
+ - Creates low-rank matrix pairs (A, B) for each selected module
+ - Tensor naming: `blk.{layer}.{module}.lora_a` and `blk.{layer}.{module}.lora_b`
+ - Dimensions: `A ∈ R^(d×r)`, `B ∈ R^(r×k)` where r is the rank
+
+2. **Weight Initialization (`llama_lora_init_tensor_weights()`):**
+ - Matrix A: Initialized with Gaussian distribution N(0, init_std)
+ - Matrix B: Initialized to zeros
+ - This ensures ΔW = BA starts at zero (no adaptation initially)
+ - Supports both CPU and GPU tensors via `ggml_backend_tensor_set()`
+ - Use `--lora-seed N` for reproducible initialization (0 = non-deterministic)
+
+3. **Buffer Allocation (`llama_lora_allocate_buffers()`):**
+ - Auto-detects backend from base model (CPU/CUDA/Vulkan)
+ - Allocates LoRA tensors on same device as model layers
+ - Uses `ggml_backend_alloc_ctx_tensors_from_buft()` for optimal placement
+
+4. **Module Selection:**
+ - Scans tensor names for patterns: `attn_q`, `attn_k`, `attn_v`, `attn_output`
+ - FFN modules: `ffn_gate`, `ffn_up`, `ffn_down`
+ - Controlled by `target_modules` bitmask (lines 194-211)
+
+5. **Optimizer Integration (`llama_opt_param_filter_lora()`):**
+ - Filter function for `ggml-opt` to identify trainable parameters
+ - Returns `true` only for tensors with `.lora_a` or `.lora_b` suffix
+ - Ensures base model weights are excluded from gradient computation
+
+6. **Checkpointing (`llama_lora_save_checkpoint()`):**
+ - Creates checkpoint directory structure
+ - Saves `model.gguf` (LoRA weights via `llama_lora_save_adapter()`)
+ - Saves `optimizer.gguf` (optimizer state via `ctx->opt_save_state()`)
+ - Both files required for resuming training
+
+**Forward Pass (`ggml-opt.cpp:ggml_opt_forward()`):**
+1. Input batch flows through base model with LoRA injections
+2. Loss computation uses only trainable LoRA parameters, we mark only the lora-parameters as trainable with `llama_opt_param_filter_lora`
+3. For instruction tuning with `--assistant-loss-only`, loss masking is applied to system/user tokens
+
+**Backward Pass (`ggml-opt.cpp:ggml_opt_backward()`):**
+1. Gradients computed only for LoRA adapters (matrices A and B)
+2. Base model weights are excluded from gradient computation
+3. Memory efficient: only stores gradients for low-rank matrices
+
+**Optimizer State (`ggml-opt.cpp`):**
+- Uses AdamW optimizer by default
+- Maintains first moment (momentum) and second moment (variance) for each LoRA parameter
+- State tensors: `opt_state_m` and `opt_state_v` for each adapter matrix
+- Checkpoint format includes full optimizer state for seamless resumption
+
+### Adding Support for a New Model Architecture in llama.cpp
+
+Supporting a new Transformer model in llama.cpp is not automatic — it requires
+implementing missing operators, adding tokenizer + prompt formatting, and
+validating training/inference parity with a reference framework. The process is
+repeatable and model families like Gemma, Qwen, Mistral, Grok, Phi can be
+onboarded by following a consistent workflow.
+
+Below is a generalized end-to-end porting procedure, derived from our work enabling
+Gemma/Qwen inference and LoRA finetuning across CPU, Vulkan, Metal, and CUDA backends.
+
+#### Step 1 — Analyze the Architecture
+
+| Component | Example Differences Across Models |
+| ------------------- | ----------------------------------------------------- |
+| Attention structure | Grouped-QKV, Multi-query, Sliding-window, Flash-attn |
+| FFN type | SiLU-MLP (LLaMA), GEGLU (Gemma), SWIGLU/MoE (Mixtral) |
+| Positional encoding | RoPE, XPos, ALiBi |
+| Tokenizer format | BPE, SentencePiece, Unigram |
+| Chat/Prompt style | ChatML, Gemini-style blocks, Turn-format |
+
+If the model deviates from LLaMA in any FFN or attention component — new ops will be required.
+
+#### Step 2 — Implement Missing Operators
+
+Almost all new model bring at least one of these requirements:
+
+| Op Type | Purpose | Required for |
+| ------------------------ | ------------------------------- | ---------------------- |
+| Feed-forward activations | GEGLU, SWIGLU, MoE dispatch | inference + training |
+| Loss + grad kernels | CROSS_ENTROPY_BACK, MASKED_LOSS | LoRA/SFT training |
+| Matrix update ops | OUT_PROD, OUT_PROD_BACK | LoRA parameter updates |
+
+#### Step 3 — Backend Kernel Extension
+
+Each operator must exist in at least one backend to work — but training must support
+Vulkan, CUDA, CPU, optionally Metal.
+
+| Backend | Required For |
+| ------- | ---------------------------------------------------------- |
+| CPU | reference correctness + unit tests |
+| Vulkan | cross-platform inference + LoRA (Adreno, Mali, AMD, Intel) |
+| CUDA | high throughput training |
+| Metal | iOS / Apple Silicon finetuning |
+
+Special attention for mobile Vulkan:
+
+- operators must tile to fit GPU SSBO memory windows
+- OUT_PROD + MUL_MAT need dynamic splitting
+- quantized INT4/INT8 variants reduce VRAM footprint dramatically
+
+#### Step 4 — Add Tokenizer, Prompt Format, Chat Template
+
+Even if inference works, instruction finetuning will fail without chat formatting.
+
+You must implement:
+
+```
+tokenizer.json / spm.model: convert to tokenizer.gguf
+default chat.jinja: system/user/assistant roles
+assistant-only masking: loss applies only to assistant tokens
+```
+
+Then train:
+
+```bash
+./llama-finetune-lora -m newmodel.gguf -f data.jsonl \
+ --assistant-loss-only --chat-template template.jinja \
+ --lora-rank 16 -ngl 999
+```
+
+#### Step 5 — Validation Workflow
+
+Before claiming model support:
+
+| Test | Pass Criteria |
+| ------------------------- | ----------------------------------------------- |
+| Generate text | No NaNs, stable token distribution |
+| Forward-only parity | Output ≈ PyTorch within float tolerance |
+| 50–200 step LoRA run | Loss decreases consistently |
+| Merge-adapter → inference | identical behavior to runtime adapter injection |
+| Finetune resumption | checkpoint restore reproducible |
+
+If inference works but training diverges, most of the time it's a missing backward op.
+
+#### Example — Adding Support for Gemma (Inference + LoRA)
+
+Gemma is a non-LLaMA architecture requiring GEGLU feed-forward layers, which means
+new forward + backward operators must be implemented before LoRA finetuning becomes
+functional. The reference implementation for this exists in the [Gemma integration
+PR](https://github.com/tetherto/qvac-ext-lib-llama.cpp/pull/63).
+
+This PR demonstrates a complete integration path: inference, instruction fine-tuning,
+adapter merging, making it an ideal template when porting additional architectures.
+
+### Troubleshooting
+
+- **Out of memory**: Reduce context length (`-c 256`), lower rank, or use fewer target modules
+- **Poor quality**: Increase rank, add more target modules, or train longer
+- **Large adapter**: Reduce rank or limit target modules
+- **Checkpoint issues**: Ensure checkpoint directory contains all required files (`model.gguf`, `optimizer.gguf`, `metadata.txt`)
+
+### Help
+
+Run with `--help` or `-h` to see all available parameters:
+```sh
+./build/bin/llama-finetune-lora --help
+```
diff --git a/examples/training/finetune-lora.cpp b/examples/training/finetune-lora.cpp
new file mode 100644
index 000000000000..40430d1a7a09
--- /dev/null
+++ b/examples/training/finetune-lora.cpp
@@ -0,0 +1,993 @@
+#include "arg.h"
+#include "common.h"
+#include "log.h"
+#include "llama.h"
+#include "ggml-backend.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#if defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
+#endif
+
+
+struct checkpoint_callback_data;
+static checkpoint_callback_data* g_checkpoint_data = nullptr;
+
+enum class lora_lr_schedule_type : std::uint8_t {
+ CONSTANT,
+ COSINE,
+ LINEAR,
+};
+
+// TODO: Ideally, training configuration variables should be added to common.h and
+// parsed using the existing common_params_parse (or loaded from a config file)
+// to reuse the existing parser and reduce boilerplate CLI parsing code.
+struct lora_lr_scheduler_state {
+ lora_lr_schedule_type schedule = lora_lr_schedule_type::CONSTANT;
+ float lr_init = 1e-5f;
+ float lr_min = 0.0f;
+ float weight_decay = 0.0f;
+ int64_t total_steps = 0;
+ int64_t current_step = 0;
+ float last_lr = 0.0f;
+ float warmup_ratio = 0.0f;
+ int64_t warmup_steps = 0;
+};
+
+static bool lora_lr_scheduler_type_from_string(const std::string & name, lora_lr_schedule_type & out) {
+ auto equals = [](const std::string & lhs, const char * rhs) {
+ const size_t rhs_len = std::strlen(rhs);
+ if (lhs.size() != rhs_len) {
+ return false;
+ }
+ for (size_t i = 0; i < rhs_len; ++i) {
+ if (std::tolower(static_cast(lhs[i])) !=
+ std::tolower(static_cast(rhs[i]))) {
+ return false;
+ }
+ }
+ return true;
+ };
+
+ if (equals(name, "constant")) {
+ out = lora_lr_schedule_type::CONSTANT;
+ return true;
+ }
+ if (equals(name, "cosine")) {
+ out = lora_lr_schedule_type::COSINE;
+ return true;
+ }
+ if (equals(name, "linear")) {
+ out = lora_lr_schedule_type::LINEAR;
+ return true;
+ }
+ return false;
+}
+
+static const char * lora_lr_scheduler_type_to_cstr(lora_lr_schedule_type type) {
+ switch (type) {
+ case lora_lr_schedule_type::LINEAR: return "linear";
+ case lora_lr_schedule_type::COSINE: return "cosine";
+ case lora_lr_schedule_type::CONSTANT: return "constant";
+ }
+ return "constant";
+}
+
+static float lora_scheduler_lr_for_step(const lora_lr_scheduler_state & state, int64_t step) {
+
+ if (state.total_steps <= 0) {
+ return std::max(state.lr_init, 0.0f);
+ }
+
+ const int64_t clamped_step = std::min(std::max(step, 0), state.total_steps);
+ const int64_t warmup_steps = std::min(std::max(state.warmup_steps, 0), state.total_steps);
+
+ if (warmup_steps > 0 && clamped_step < warmup_steps) {
+ const float warmup_progress = static_cast(clamped_step) / static_cast(warmup_steps);
+ const float lr = state.lr_init * warmup_progress;
+ return std::max(lr, 0.0f);
+ }
+
+ const int64_t adjusted_step = clamped_step - warmup_steps;
+ int64_t remaining_steps = state.total_steps - warmup_steps;
+ if (remaining_steps <= 0) {
+ remaining_steps = 1;
+ }
+
+ const float progress = std::min(static_cast(adjusted_step) / static_cast(remaining_steps), 1.0f);
+ float lr = state.lr_init;
+
+ switch (state.schedule) {
+ case lora_lr_schedule_type::CONSTANT:
+ lr = state.lr_init;
+ break;
+ case lora_lr_schedule_type::COSINE: {
+ constexpr float kPi = 3.14159265358979323846f;
+ const float cosine = 0.5f * (1.0f + std::cos(progress * kPi));
+ lr = state.lr_min + (state.lr_init - state.lr_min) * cosine;
+ break;
+ }
+ case lora_lr_schedule_type::LINEAR: {
+ lr = state.lr_init + (state.lr_min - state.lr_init) * progress;
+ break;
+ }
+ }
+
+ return std::max(lr, 0.0f);
+}
+
+static struct ggml_opt_optimizer_params lora_scheduler_get_optimizer_params(void * userdata) {
+ auto * scheduler = static_cast(userdata);
+ struct ggml_opt_optimizer_params params = ggml_opt_get_default_optimizer_params(nullptr);
+
+ if (!scheduler) {
+ return params;
+ }
+
+ const float lr = lora_scheduler_lr_for_step(*scheduler, scheduler->current_step+1);
+ scheduler->last_lr = lr;
+
+ params.adamw.alpha = lr;
+ params.adamw.wd = scheduler->weight_decay;
+
+ params.sgd.alpha = lr;
+ params.sgd.wd = scheduler->weight_decay;
+
+ if (scheduler->current_step < scheduler->total_steps) {
+ scheduler->current_step++;
+ }
+
+ return params;
+}
+
+static uint32_t parse_lora_modules(const std::string& modules_str) {
+ if (modules_str.empty()) {
+ return LLAMA_LORA_TARGET_ATTN_Q | LLAMA_LORA_TARGET_ATTN_K | LLAMA_LORA_TARGET_ATTN_V | LLAMA_LORA_TARGET_ATTN_O;
+ }
+
+ static const std::map module_map = {
+ {"attn_q", LLAMA_LORA_TARGET_ATTN_Q},
+ {"attn_k", LLAMA_LORA_TARGET_ATTN_K},
+ {"attn_v", LLAMA_LORA_TARGET_ATTN_V},
+ {"attn_o", LLAMA_LORA_TARGET_ATTN_O},
+ {"ffn_gate", LLAMA_LORA_TARGET_FFN_GATE},
+ {"ffn_up", LLAMA_LORA_TARGET_FFN_UP},
+ {"ffn_down", LLAMA_LORA_TARGET_FFN_DOWN},
+ {"output", LLAMA_LORA_TARGET_OUTPUT},
+ {"all", LLAMA_LORA_TARGET_ALL}
+ };
+
+ uint32_t target_modules = 0;
+ std::stringstream ss(modules_str);
+ std::string module;
+
+ while (std::getline(ss, module, ',')) {
+ module.erase(0, module.find_first_not_of(" \t"));
+ module.erase(module.find_last_not_of(" \t") + 1);
+
+ auto it = module_map.find(module);
+ if (it != module_map.end()) {
+ target_modules |= it->second;
+ LOG_INF("Added target module: %s\n", module.c_str());
+ } else {
+ LOG_ERR("Unknown LoRA target module: %s\n", module.c_str());
+ LOG_ERR("Available modules: attn_q, attn_k, attn_v, attn_o, ffn_gate, ffn_up, ffn_down, output, all\n");
+ return 0;
+ }
+ }
+
+ return target_modules;
+}
+
+static bool training_supports_out_prod_f16(const common_params & params) {
+ std::vector devices;
+
+ if (!params.devices.empty()) {
+ devices.assign(params.devices.begin(), params.devices.end());
+ } else {
+ ggml_backend_dev_t gpu = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
+ if (gpu) {
+ devices.push_back(gpu);
+ }
+ }
+
+ if (devices.empty()) {
+ return true;
+ }
+
+ constexpr int64_t ne0 = 4;
+ constexpr int64_t ne1 = 3;
+ constexpr int64_t k = 2;
+
+ struct ggml_tensor src0 = {};
+ struct ggml_tensor src1 = {};
+ struct ggml_tensor dst = {};
+
+ src0.type = GGML_TYPE_F16;
+ src1.type = GGML_TYPE_F32;
+ dst.type = GGML_TYPE_F32;
+
+ src0.ne[0] = ne0; src0.ne[1] = k; src0.ne[2] = 1; src0.ne[3] = 1;
+ src1.ne[0] = ne1; src1.ne[1] = k; src1.ne[2] = 1; src1.ne[3] = 1;
+ dst.ne [0] = ne0; dst.ne [1] = ne1; dst.ne [2] = 1; dst.ne [3] = 1;
+
+ src0.nb[0] = sizeof(ggml_fp16_t);
+ src0.nb[1] = src0.nb[0] * ne0;
+ src0.nb[2] = src0.nb[1] * k;
+ src0.nb[3] = src0.nb[2] * 1;
+
+ src1.nb[0] = sizeof(float);
+ src1.nb[1] = src1.nb[0] * ne1;
+ src1.nb[2] = src1.nb[1] * k;
+ src1.nb[3] = src1.nb[2] * 1;
+
+ dst.nb[0] = sizeof(float);
+ dst.nb[1] = dst.nb[0] * ne0;
+ dst.nb[2] = dst.nb[1] * ne1;
+ dst.nb[3] = dst.nb[2] * 1;
+
+ dst.op = GGML_OP_OUT_PROD;
+ dst.src[0] = &src0;
+ dst.src[1] = &src1;
+
+ for (ggml_backend_dev_t dev : devices) {
+ if (dev == nullptr) {
+ continue;
+ }
+ if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
+ continue;
+ }
+ if (!ggml_backend_dev_supports_op(dev, &dst)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static void print_lora_usage() {
+ printf("\n----- LoRA Fine-tuning Parameters -----\n");
+ printf(" --lora-rank N LoRA rank (default: 8, range: 1-512)\n");
+ printf(" --lora-alpha N LoRA alpha scaling factor (default: 16.0, range: 0.1-1000.0)\n");
+ printf(" --lora-modules MODULES Target modules as comma-separated list (default: attn_q,attn_k,attn_v,attn_o)\n");
+ printf(" Available modules: attn_q, attn_k, attn_v, attn_o, ffn_gate, ffn_up, ffn_down, output, all\n");
+ printf(" Examples: \"attn_q,attn_v\" or \"all\" or \"attn_q,attn_k,attn_v,attn_o,ffn_gate,ffn_up,ffn_down\"\n");
+ printf(" --output-adapter PATH Output path for trained adapter (default: auto-generated)\n");
+ printf("\nTraining Options:\n");
+ printf(" --num-epochs N Number of training epochs (default: 1)\n");
+ printf(" --assistant-loss-only Use JSON dataset format with masked loss (ChatML/conversation format)\n");
+ printf(" Only computes loss on assistant responses, not system/user prompts\n");
+ printf(" --chat-template PATH Optional Jinja chat template to render JSON dataset (matches HF apply_chat_template)\n");
+ printf(" --learning-rate F AdamW learning rate (default: 1e-5)\n");
+ printf(" --weight-decay F AdamW weight decay (default: 1e-2)\n");
+ printf(" --lr-scheduler TYPE Learning rate scheduler: constant, cosine, linear (default: constant)\n");
+ printf(" --lr-min F Minimum LR for cosine/linear schedulers (default: 0)\n");
+ printf(" --warmup-ratio F Fraction of total steps for LR warmup (default: 0.0)\n");
+ printf(" --warmup-steps N Explicit warmup steps (overrides warmup-ratio)\n");
+ printf("\nCheckpointing Options:\n");
+ printf(" --checkpoint-save-steps N Save checkpoint every N training steps (default: 100)\n");
+ printf(" --checkpoint-save-dir PATH Directory for checkpoints (default: ./checkpoints)\n");
+ printf(" --resume-from PATH Resume training from specific checkpoint file\n");
+ printf(" --auto-resume Automatically resume from latest checkpoint in save dir\n");
+ printf("\nExamples:\n");
+ printf(" # Train with rank=16, alpha=32, all attention modules\n");
+ printf(" %s -m model.gguf -f dataset.txt --lora-rank 16 --lora-alpha 32 --lora-modules attn_q,attn_k,attn_v,attn_o\n", "finetune-lora");
+ printf("\n # Fine-tune existing adapter with all modules\n");
+ printf(" %s -m model.gguf -f dataset.txt --lora existing.gguf --output-adapter improved.gguf\n", "finetune-lora");
+ printf("\n # Instruction fine-tuning with ChatML format\n");
+ printf(" %s -m model.gguf -f conversations.jsonl --assistant-loss-only --lora-rank 16\n", "finetune-lora");
+ printf("\n");
+}
+
+struct checkpoint_metadata {
+ int32_t epoch;
+ int32_t lora_rank;
+ float lora_alpha;
+ uint32_t target_modules;
+};
+
+static std::string get_checkpoint_filename(const std::string& checkpoint_dir, int64_t step) {
+ std::ostringstream oss;
+ oss << checkpoint_dir << "/checkpoint_step_" << std::setfill('0') << std::setw(8) << step;
+ return oss.str();
+}
+
+static std::string find_latest_checkpoint(const std::string& checkpoint_dir) {
+ if (!std::filesystem::exists(checkpoint_dir)) {
+ return "";
+ }
+
+ std::string latest_checkpoint;
+ int64_t latest_step = -1;
+
+ for (const auto& entry : std::filesystem::directory_iterator(checkpoint_dir)) {
+ if (entry.is_directory()) {
+ std::string dirname = entry.path().filename().string();
+ if (dirname.find("checkpoint_step_") == 0 && dirname.size() >= 16) {
+ std::string step_str = dirname.substr(16, 8);
+ try {
+ int64_t step = std::stoll(step_str);
+ if (step > latest_step) {
+ latest_step = step;
+ latest_checkpoint = entry.path().string();
+ }
+ } catch (const std::exception&) {
+ continue;
+ }
+ }
+ }
+ }
+
+ return latest_checkpoint;
+}
+
+static bool save_checkpoint(llama_context* ctx, llama_adapter_lora* adapter, const checkpoint_metadata& metadata, const std::string& checkpoint_dir) {
+ if (!std::filesystem::exists(checkpoint_dir)) {
+ if (!std::filesystem::create_directories(checkpoint_dir)) {
+ LOG_ERR("Failed to create checkpoint directory: %s\n", checkpoint_dir.c_str());
+ return false;
+ }
+ }
+
+ if (!llama_lora_save_checkpoint(adapter, checkpoint_dir.c_str(), llama_get_model(ctx), ctx)) {
+ LOG_ERR("Failed to save LoRA checkpoint\n");
+ return false;
+ }
+
+ std::string meta_path = checkpoint_dir + "/metadata.txt";
+ std::ofstream meta_file(meta_path);
+ if (meta_file.is_open()) {
+ meta_file << "epoch=" << metadata.epoch << "\n";
+ meta_file << "lora_rank=" << metadata.lora_rank << "\n";
+ meta_file << "lora_alpha=" << metadata.lora_alpha << "\n";
+ meta_file << "target_modules=" << metadata.target_modules << "\n";
+ meta_file.close();
+ } else {
+ LOG_ERR("Failed to save checkpoint metadata\n");
+ return false;
+ }
+
+ LOG_INF("Checkpoint saved successfully to %s\n", checkpoint_dir.c_str());
+ return true;
+}
+
+static bool validate_checkpoint_metadata(const std::string& checkpoint_path, checkpoint_metadata& metadata) {
+ std::string checkpoint_dir = checkpoint_path;
+
+ if (!std::filesystem::exists(checkpoint_dir)) {
+ LOG_ERR("Checkpoint directory does not exist: %s\n", checkpoint_dir.c_str());
+ return false;
+ }
+
+ LOG_INF("Loading checkpoint from: %s\n", checkpoint_dir.c_str());
+
+ std::string meta_path = checkpoint_dir + "/metadata.txt";
+ if (std::filesystem::exists(meta_path)) {
+ std::ifstream meta_file(meta_path);
+ if (meta_file.is_open()) {
+ std::string line;
+ while (std::getline(meta_file, line)) {
+ size_t eq_pos = line.find('=');
+ if (eq_pos != std::string::npos) {
+ std::string key = line.substr(0, eq_pos);
+ std::string value = line.substr(eq_pos + 1);
+
+ if (key == "epoch") {
+ metadata.epoch = std::stoi(value);
+ } else if (key == "lora_rank") {
+ metadata.lora_rank = std::stoi(value);
+ } else if (key == "lora_alpha") {
+ metadata.lora_alpha = std::stof(value);
+ } else if (key == "target_modules") {
+ metadata.target_modules = std::stoul(value);
+ }
+ }
+ }
+ meta_file.close();
+ } else {
+ LOG_ERR("Failed to open checkpoint metadata file\n");
+ return false;
+ }
+ } else {
+ LOG_ERR("Checkpoint metadata file not found: %s\n", meta_path.c_str());
+ return false;
+ }
+
+ LOG_INF("Checkpoint loaded successfully\n");
+ return true;
+}
+
+
+struct checkpoint_callback_data {
+ llama_context* ctx;
+ llama_adapter_lora* adapter;
+ int32_t checkpoint_save_steps;
+ std::string checkpoint_save_dir;
+ int64_t global_step;
+ int64_t initial_step;
+ int32_t current_epoch;
+ int32_t lora_rank;
+ float lora_alpha;
+ uint32_t target_modules;
+ float learning_rate;
+ lora_lr_scheduler_state * lr_scheduler;
+ std::string model_path;
+ std::string dataset_path;
+};
+
+static void checkpoint_progress_callback(
+ bool train,
+ ggml_opt_context_t opt_ctx,
+ ggml_opt_dataset_t dataset,
+ ggml_opt_result_t result,
+ int64_t ibatch,
+ int64_t ibatch_max,
+ int64_t t_start_us) {
+ ggml_opt_epoch_callback_progress_bar(train, opt_ctx, dataset, result, ibatch, ibatch_max, t_start_us);
+
+ if (!train) {
+ return;
+ }
+
+ checkpoint_callback_data* cb_data = g_checkpoint_data;
+
+ if (!cb_data) {
+ LOG_ERR("Checkpoint callback data is null!\n");
+ return;
+ }
+
+ if (cb_data->lr_scheduler) {
+ cb_data->learning_rate = lora_scheduler_lr_for_step(*cb_data->lr_scheduler, cb_data->lr_scheduler->current_step+1);
+ }
+
+ if (cb_data->checkpoint_save_steps <= 0) {
+ return;
+ }
+
+ cb_data->global_step++;
+
+ if (cb_data->global_step % cb_data->checkpoint_save_steps == 0) {
+ if (!cb_data->ctx) {
+ LOG_ERR("Context is null in checkpoint callback!\n");
+ return;
+ }
+
+ if (!cb_data->adapter) {
+ LOG_ERR("LoRA adapter is null in checkpoint callback!\n");
+ return;
+ }
+
+ checkpoint_metadata meta = {
+ /*epoch =*/ cb_data->current_epoch,
+ /*lora_rank =*/ cb_data->lora_rank,
+ /*lora_alpha =*/ cb_data->lora_alpha,
+ /*target_modules =*/ cb_data->target_modules,
+ };
+
+ std::string checkpoint_path = get_checkpoint_filename(cb_data->checkpoint_save_dir, cb_data->global_step);
+
+ if (!save_checkpoint(cb_data->ctx, cb_data->adapter, meta, checkpoint_path)) {
+ LOG_ERR("Failed to save checkpoint at step %" PRId64 "\n", cb_data->global_step);
+ }
+ }
+}
+
+struct finetune_params {
+ int32_t lora_rank = 8;
+ float lora_alpha = 16.0f;
+ std::string lora_modules_str;
+ std::string output_adapter_path;
+
+ int32_t num_epochs = 1;
+ float learning_rate = 1e-5f;
+ float lr_min = 0.0f;
+ float weight_decay = 0.01f;
+ std::string lr_scheduler = "constant";
+ float warmup_ratio = 0.0f;
+ int64_t warmup_steps = 0;
+ bool warmup_ratio_set = false;
+ bool warmup_steps_set = false;
+
+ int32_t checkpoint_save_steps = 100;
+ std::string checkpoint_save_dir = "./checkpoints";
+ std::string resume_from_checkpoint;
+ bool auto_resume = false;
+ std::string chat_template_path;
+ bool assistant_loss_only = false;
+ uint32_t lora_seed = 42;
+};
+
+static bool parse_finetune_args(int& argc, char** argv, finetune_params& ft_params) {
+ auto remove_arg_pair = [&](int i) {
+ for (int j = i; j < argc - 2; j++) {
+ argv[j] = argv[j + 2];
+ }
+ argc -= 2;
+ };
+
+ auto remove_arg_single = [&](int i) {
+ for (int j = i; j < argc - 1; j++) {
+ argv[j] = argv[j + 1];
+ }
+ argc -= 1;
+ };
+
+ for (int i = 1; i < argc; i++) {
+ if (strcmp(argv[i], "--assistant-loss-only") == 0) {
+ ft_params.assistant_loss_only = true;
+ remove_arg_single(i);
+ i--;
+ }
+ }
+
+ for (int i = 1; i < argc; i++) {
+ if (strcmp(argv[i], "--lora-rank") == 0 && i + 1 < argc) {
+ ft_params.lora_rank = std::atoi(argv[i + 1]);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--lora-alpha") == 0 && i + 1 < argc) {
+ ft_params.lora_alpha = std::atof(argv[i + 1]);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--lora-seed") == 0 && i + 1 < argc) {
+ ft_params.lora_seed = std::strtoul(argv[i + 1], nullptr, 10);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--lora-modules") == 0 && i + 1 < argc) {
+ ft_params.lora_modules_str = argv[i + 1];
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--output-adapter") == 0 && i + 1 < argc) {
+ ft_params.output_adapter_path = argv[i + 1];
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--num-epochs") == 0 && i + 1 < argc) {
+ ft_params.num_epochs = std::atoi(argv[i + 1]);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--learning-rate") == 0 && i + 1 < argc) {
+ ft_params.learning_rate = std::atof(argv[i + 1]);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--weight-decay") == 0 && i + 1 < argc) {
+ ft_params.weight_decay = std::atof(argv[i + 1]);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--lr-scheduler") == 0 && i + 1 < argc) {
+ ft_params.lr_scheduler = argv[i + 1];
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--lr-min") == 0 && i + 1 < argc) {
+ ft_params.lr_min = std::atof(argv[i + 1]);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--warmup-ratio") == 0 && i + 1 < argc) {
+ ft_params.warmup_ratio = std::atof(argv[i + 1]);
+ ft_params.warmup_ratio_set = true;
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--warmup-steps") == 0 && i + 1 < argc) {
+ ft_params.warmup_steps = std::atoll(argv[i + 1]);
+ ft_params.warmup_steps_set = true;
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--checkpoint-save-steps") == 0 && i + 1 < argc) {
+ ft_params.checkpoint_save_steps = std::atoi(argv[i + 1]);
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--checkpoint-save-dir") == 0 && i + 1 < argc) {
+ ft_params.checkpoint_save_dir = argv[i + 1];
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--resume-from") == 0 && i + 1 < argc) {
+ ft_params.resume_from_checkpoint = argv[i + 1];
+ remove_arg_pair(i);
+ i--;
+ } else if (strcmp(argv[i], "--auto-resume") == 0) {
+ ft_params.auto_resume = true;
+ for (int j = i; j < argc - 1; j++) {
+ argv[j] = argv[j + 1];
+ }
+ argc--;
+ i--;
+ } else if (strcmp(argv[i], "--chat-template") == 0 && i + 1 < argc) {
+ ft_params.chat_template_path = argv[i + 1];
+ remove_arg_pair(i);
+ i--;
+ }
+ }
+
+ for (int i = 1; i < argc; i++) {
+ if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+ print_lora_usage();
+ }
+ }
+
+ return true;
+}
+
+int main(int argc, char ** argv) {
+ common_params params;
+ finetune_params ft_params;
+
+ params.escape = false;
+ parse_finetune_args(argc, argv, ft_params);
+
+ if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_PERPLEXITY)) {
+ return 1;
+ }
+
+ lora_lr_schedule_type scheduler_type;
+ if (!lora_lr_scheduler_type_from_string(ft_params.lr_scheduler, scheduler_type)) {
+ LOG_ERR("Unknown learning rate scheduler: %s (expected: constant, cosine, linear)\n", ft_params.lr_scheduler.c_str());
+ return 1;
+ }
+
+ if (ft_params.num_epochs <= 0) {
+ LOG_ERR("Number of epochs must be > 0, got %d\n", ft_params.num_epochs);
+ return 1;
+ }
+ if (ft_params.learning_rate <= 0.0f) {
+ LOG_ERR("Learning rate must be > 0, got %.4e\n", ft_params.learning_rate);
+ return 1;
+ }
+ if (ft_params.weight_decay < 0.0f) {
+ LOG_ERR("Weight decay must be >= 0, got %.4e\n", ft_params.weight_decay);
+ return 1;
+ }
+ if (ft_params.lr_min < 0.0f) {
+ LOG_ERR("Minimum learning rate must be >= 0, got %.4e\n", ft_params.lr_min);
+ return 1;
+ }
+ const bool scheduler_uses_lr_min = scheduler_type == lora_lr_schedule_type::COSINE ||
+ scheduler_type == lora_lr_schedule_type::LINEAR;
+ if (scheduler_uses_lr_min && ft_params.lr_min > ft_params.learning_rate) {
+ LOG_ERR("For %s scheduler lr-min (%.4e) cannot exceed learning-rate (%.4e)\n",
+ lora_lr_scheduler_type_to_cstr(scheduler_type), ft_params.lr_min, ft_params.learning_rate);
+ return 1;
+ }
+
+ LOG_INF("Using LoRA parameters: rank=%d, alpha=%.1f, seed=%u\n", ft_params.lora_rank, ft_params.lora_alpha, ft_params.lora_seed);
+ LOG_INF("Training for %d epochs\n", ft_params.num_epochs);
+
+ // Handle checkpoint auto-resume before model initialization
+ if (ft_params.auto_resume && ft_params.resume_from_checkpoint.empty()) {
+ std::string latest_checkpoint = find_latest_checkpoint(ft_params.checkpoint_save_dir);
+ if (!latest_checkpoint.empty()) {
+ ft_params.resume_from_checkpoint = latest_checkpoint;
+ LOG_INF("Auto-resume: found checkpoint %s\n", ft_params.resume_from_checkpoint.c_str());
+ }
+ }
+
+ if (!ft_params.resume_from_checkpoint.empty()) {
+ params.warmup = false;
+ }
+
+ // Load checkpoint LoRA adapter from directory structure (model.gguf)
+ if (!ft_params.resume_from_checkpoint.empty()) {
+ std::filesystem::path checkpoint_dir(ft_params.resume_from_checkpoint);
+ std::filesystem::path model_path = checkpoint_dir / "model.gguf";
+
+ LOG_INF("Loading checkpoint LoRA adapter: %s\n", model_path.c_str());
+ common_adapter_lora_info lora_adapter;
+ lora_adapter.path = model_path.string();
+ lora_adapter.scale = 1.0f;
+ lora_adapter.ptr = nullptr;
+ params.lora_adapters.clear(); // Remove any existing adapters
+ params.lora_adapters.push_back(lora_adapter);
+ LOG_INF("Checkpoint LoRA adapter added to params\n");
+ }
+
+ if (params.use_mmap) {
+ LOG_INF("%s: force disabling memory mapping because it would result in-read-only pointers to the weights\n", __func__);
+ params.use_mmap = false;
+ }
+ const bool supports_out_prod_f16 = training_supports_out_prod_f16(params);
+ if (!supports_out_prod_f16) {
+ if (params.cache_type_k != GGML_TYPE_F32) {
+ LOG_INF("%s: force changing k cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
+ params.cache_type_k = GGML_TYPE_F32;
+ }
+ if (params.cache_type_v != GGML_TYPE_F32) {
+ LOG_INF("%s: force changing v cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
+ params.cache_type_v = GGML_TYPE_F32;
+ }
+ }
+
+ common_init();
+ llama_backend_init();
+ llama_numa_init(params.numa);
+ params.training = true;
+
+ common_init_result llama_init = common_init_from_params(params);
+ llama_model_ptr & model = llama_init.model;
+ llama_context_ptr & ctx = llama_init.context;
+
+ if (model == NULL) {
+ LOG_ERR("%s: unable to load model\n", __func__);
+ return 1;
+ }
+
+ {
+ LOG_INF("\n");
+ LOG_INF("%s\n", common_params_get_system_info(params).c_str());
+ }
+
+ uint32_t target_modules = parse_lora_modules(ft_params.lora_modules_str);
+ if (target_modules == 0) {
+ return 1;
+ }
+
+ struct llama_lora_training_params lora_params = {
+ /*target_modules =*/ target_modules,
+ /*rank =*/ ft_params.lora_rank,
+ /*alpha =*/ ft_params.lora_alpha,
+ /*dropout =*/ 0.0f,
+ /*init_std =*/ 0.02f,
+ /*seed =*/ ft_params.lora_seed,
+ };
+
+ bool has_existing_lora = !params.lora_adapters.empty();
+ struct llama_adapter_lora * trained_adapter = nullptr;
+
+ if (has_existing_lora) {
+ LOG_INF("Finetuning existing LoRA adapters\n");
+ LOG_INF("Found %zu existing LoRA adapters to train\n", params.lora_adapters.size());
+ trained_adapter = params.lora_adapters[0].ptr;
+ if (!trained_adapter) {
+ LOG_ERR("Existing LoRA adapter is null\n");
+ return 1;
+ }
+ } else {
+ LOG_INF("Target modules: Q=%s, K=%s, V=%s, O=%s, GATE=%s, UP=%s, DOWN=%s, OUTPUT=%s\n",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_ATTN_Q) ? "yes" : "no",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_ATTN_K) ? "yes" : "no",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_ATTN_V) ? "yes" : "no",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_ATTN_O) ? "yes" : "no",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_FFN_GATE) ? "yes" : "no",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_FFN_UP) ? "yes" : "no",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_FFN_DOWN) ? "yes" : "no",
+ (lora_params.target_modules & LLAMA_LORA_TARGET_OUTPUT) ? "yes" : "no");
+
+ LOG_INF("LoRA configuration: rank=%d, alpha=%.1f (scaling=%.3f)\n",
+ lora_params.rank, lora_params.alpha, lora_params.alpha / lora_params.rank);
+
+ trained_adapter = llama_lora_training_init(ctx.get(), model.get(), &lora_params);
+ if (!trained_adapter) {
+ LOG_ERR("%s: LoRA training initialization failed\n", __func__);
+ return 1;
+ }
+ }
+
+ constexpr float val_split = 0.05f;
+
+ ggml_opt_dataset_t dataset;
+
+ if (ft_params.assistant_loss_only) {
+ LOG_INF("Using JSON dataset with chat template and assistant-only loss\n");
+ dataset = common_opt_sft_dataset_init(ctx.get(), params.prompt, llama_n_ctx(ctx.get())/2, ft_params.chat_template_path);
+ } else {
+ std::vector tokens = common_tokenize(ctx.get(), params.prompt, true);
+ LOG_INF("Using standard next-token prediction mode\n");
+ dataset = common_opt_dataset_init(ctx.get(), tokens, llama_n_ctx(ctx.get())/2);
+ }
+
+ if (dataset == nullptr) {
+ LOG_ERR("Failed to create dataset. Please check your input file and parameters.\n");
+ return 1;
+ }
+
+ const int64_t total_datapoints = ggml_opt_dataset_ndata(dataset);
+ const int64_t idata_split = static_cast(total_datapoints * (1.0f - val_split));
+ const int64_t training_batches_per_epoch = idata_split;
+
+ if (training_batches_per_epoch <= 0) {
+ LOG_ERR("Training split is empty. Adjust --val-split or dataset size.\n");
+ return 1;
+ }
+
+ lora_lr_scheduler_state lr_scheduler;
+ lr_scheduler.schedule = scheduler_type;
+ lr_scheduler.lr_init = ft_params.learning_rate;
+ lr_scheduler.lr_min = (scheduler_type == lora_lr_schedule_type::CONSTANT) ? ft_params.learning_rate : ft_params.lr_min;
+ lr_scheduler.weight_decay = ft_params.weight_decay;
+ lr_scheduler.total_steps = std::max(1, static_cast(ft_params.num_epochs) * training_batches_per_epoch);
+ if (ft_params.warmup_steps_set) {
+ lr_scheduler.warmup_steps = std::min(ft_params.warmup_steps, lr_scheduler.total_steps);
+ } else if (ft_params.warmup_ratio_set) {
+ const double warmup_from_ratio = static_cast(lr_scheduler.total_steps) * static_cast(ft_params.warmup_ratio);
+ lr_scheduler.warmup_steps = std::min(static_cast(warmup_from_ratio), lr_scheduler.total_steps);
+ } else {
+ lr_scheduler.warmup_steps = 0;
+ }
+ lr_scheduler.warmup_steps = std::max(lr_scheduler.warmup_steps, 0);
+ if (lr_scheduler.total_steps > 0) {
+ lr_scheduler.warmup_ratio = static_cast(lr_scheduler.warmup_steps) / static_cast(lr_scheduler.total_steps);
+ } else {
+ lr_scheduler.warmup_ratio = 0.0f;
+ }
+ lr_scheduler.current_step = 0;
+ lr_scheduler.last_lr = lora_scheduler_lr_for_step(lr_scheduler, lr_scheduler.current_step);
+
+ LOG_INF("Training split: datapoints=%lld, batches_per_epoch=%lld\n",
+ (long long) total_datapoints, (long long) training_batches_per_epoch);
+ LOG_INF("Optimizer: adamw scheduler=%s lr=%.4e wd=%.4e total_steps=%lld\n",
+ lora_lr_scheduler_type_to_cstr(lr_scheduler.schedule), lr_scheduler.lr_init,
+ lr_scheduler.weight_decay, (long long) lr_scheduler.total_steps);
+ if (lr_scheduler.schedule == lora_lr_schedule_type::COSINE) {
+ LOG_INF("Cosine scheduler: lr-min=%.4e\n", lr_scheduler.lr_min);
+ } else if (lr_scheduler.schedule == lora_lr_schedule_type::LINEAR) {
+ LOG_INF("Linear scheduler: lr-min=%.4e\n", lr_scheduler.lr_min);
+ }
+ if (lr_scheduler.warmup_steps > 0) {
+ LOG_INF("Warmup: steps=%lld ratio=%.4f\n", (long long) lr_scheduler.warmup_steps, lr_scheduler.warmup_ratio);
+ } else if (ft_params.warmup_ratio_set) {
+ LOG_WRN("Warmup ratio %.4f produced 0 warmup steps (total_steps=%lld); no warmup applied\n",
+ ft_params.warmup_ratio, (long long) lr_scheduler.total_steps);
+ }
+
+ int start_epoch = 0;
+ int64_t start_step = 0;
+ checkpoint_metadata checkpoint_meta = {};
+ bool checkpoint_loaded = false;
+
+ if (!ft_params.resume_from_checkpoint.empty()) {
+ if (validate_checkpoint_metadata(ft_params.resume_from_checkpoint, checkpoint_meta)) {
+ start_epoch = checkpoint_meta.epoch;
+ checkpoint_loaded = true;
+
+ if (checkpoint_meta.lora_rank != ft_params.lora_rank) {
+ LOG_ERR("Checkpoint LoRA rank (%d) doesn't match current rank (%d). Use --resume-from to manually specify a compatible checkpoint.\n",
+ checkpoint_meta.lora_rank, ft_params.lora_rank);
+ return 1;
+ }
+ if (checkpoint_meta.lora_alpha != ft_params.lora_alpha) {
+ LOG_ERR("Checkpoint LoRA alpha (%.3f) doesn't match current alpha (%.3f)\n",
+ checkpoint_meta.lora_alpha, ft_params.lora_alpha);
+ return 1;
+ }
+ if (checkpoint_meta.target_modules != target_modules) {
+ LOG_ERR("Checkpoint target_modules doesn't match current target_modules\n");
+ return 1;
+ }
+
+ } else {
+ LOG_ERR("Failed to load checkpoint, starting from scratch\n");
+ }
+ }
+
+ std::string optimizer_checkpoint_path;
+ if (checkpoint_loaded && !ft_params.resume_from_checkpoint.empty()) {
+ std::filesystem::path checkpoint_dir(ft_params.resume_from_checkpoint);
+ optimizer_checkpoint_path = (checkpoint_dir / "optimizer.gguf").string();
+ }
+
+ struct llama_opt_params lopt_params = llama_opt_default_params();
+ lopt_params.param_filter = llama_opt_param_filter_lora;
+ lopt_params.get_opt_pars = lora_scheduler_get_optimizer_params;
+ lopt_params.get_opt_pars_ud = &lr_scheduler;
+ lopt_params.checkpoint_path = checkpoint_loaded ? optimizer_checkpoint_path.c_str() : nullptr;
+ lopt_params.load_optimizer_state = checkpoint_loaded;
+ lopt_params.assistant_loss_only = ft_params.assistant_loss_only;
+
+ llama_opt_init(ctx.get(), model.get(), lopt_params);
+
+ if (checkpoint_loaded) {
+ start_step = llama_opt_get_iter(ctx.get());
+ }
+
+ lr_scheduler.current_step = std::min(start_step, lr_scheduler.total_steps);
+ lr_scheduler.last_lr = lora_scheduler_lr_for_step(lr_scheduler, lr_scheduler.current_step);
+
+ if (!trained_adapter) {
+ LOG_ERR("No trained adapter available for checkpointing\n");
+ return 1;
+ }
+
+ if (start_step > 0) {
+ int64_t completed_epochs = start_step / training_batches_per_epoch;
+ start_epoch = (int)completed_epochs;
+ LOG_INF("Resuming training from global step %lld (lr=%.4e)\n",
+ (long long) start_step, lr_scheduler.last_lr);
+ }
+
+ checkpoint_callback_data cb_data = {
+ /*ctx =*/ ctx.get(),
+ /*adapter =*/ trained_adapter,
+ /*checkpoint_save_steps =*/ ft_params.checkpoint_save_steps,
+ /*checkpoint_save_dir =*/ ft_params.checkpoint_save_dir,
+ /*global_step =*/ start_step,
+ /*initial_step =*/ start_step,
+ /*current_epoch =*/ start_epoch,
+ /*lora_rank =*/ ft_params.lora_rank,
+ /*lora_alpha =*/ ft_params.lora_alpha,
+ /*target_modules =*/ target_modules,
+ /*learning_rate =*/ lr_scheduler.last_lr,
+ /*lr_scheduler =*/ &lr_scheduler,
+ /*model_path =*/ params.model.path,
+ /*dataset_path =*/ params.prompt_file,
+ };
+ g_checkpoint_data = &cb_data;
+
+ ggml_opt_result_t result_train = ggml_opt_result_init();
+ ggml_opt_result_t result_eval = ggml_opt_result_init();
+
+ for (int epoch = start_epoch; epoch < ft_params.num_epochs; ++epoch) {
+ if (cb_data.lr_scheduler) {
+ cb_data.learning_rate = lora_scheduler_lr_for_step(*cb_data.lr_scheduler, cb_data.lr_scheduler->current_step+1);
+ }
+ LOG_INF("Starting epoch %d (step %lld, lr=%.4e)\n", epoch, (long long)cb_data.global_step, cb_data.learning_rate);
+ cb_data.current_epoch = epoch;
+
+ int64_t resume_batch = 0;
+ if (start_step > 0 && epoch == start_epoch) {
+ resume_batch = start_step % training_batches_per_epoch;
+ }
+
+ ggml_opt_epoch_callback train_callback = (ft_params.checkpoint_save_steps <= 0) ?
+ ggml_opt_epoch_callback_progress_bar : checkpoint_progress_callback;
+ ggml_opt_epoch_callback eval_callback = (ft_params.checkpoint_save_steps <= 0) ?
+ ggml_opt_epoch_callback_progress_bar : checkpoint_progress_callback;
+
+ if (resume_batch > 0) {
+ LOG_INF("Resuming training from epoch %d, step %" PRId64 " \n", epoch, resume_batch);
+ } else if (ft_params.checkpoint_save_steps > 0) {
+ LOG_INF("Checkpointing enabled, saving every %d steps\n", ft_params.checkpoint_save_steps);
+ } else {
+ LOG_INF("Checkpointing disabled, using standard progress callback\n");
+ }
+
+ llama_opt_epoch_resume(ctx.get(), dataset, result_train, result_eval, idata_split,
+ train_callback, eval_callback, resume_batch);
+ fprintf(stderr, "\n");
+
+ ggml_opt_result_reset(result_train);
+ ggml_opt_result_reset(result_eval);
+ }
+
+ g_checkpoint_data = nullptr;
+ ggml_opt_result_free(result_train);
+ ggml_opt_result_free(result_eval);
+
+ std::string adapter_filename;
+ if (!ft_params.output_adapter_path.empty()) {
+ adapter_filename = ft_params.output_adapter_path;
+ } else if (has_existing_lora) {
+ adapter_filename = "finetuned-lora-adapter.gguf";
+ LOG_INF("Finetuned existing lora adapter, saving as: %s\n", adapter_filename.c_str());
+ } else {
+ adapter_filename = "trained-lora-adapter.gguf";
+ LOG_INF("Saving new lora adapter: %s\n", adapter_filename.c_str());
+ }
+
+ if (trained_adapter) {
+ if (llama_lora_save_adapter(trained_adapter, adapter_filename.c_str(), model.get())) {
+ std::ifstream adapter_file(adapter_filename, std::ios::binary | std::ios::ate);
+ if (adapter_file.is_open()) {
+ std::streamsize adapter_size = adapter_file.tellg();
+ LOG_INF("LoRA adapter saved: %s (%.2f MB)\n",
+ adapter_filename.c_str(), adapter_size / (1024.0 * 1024.0));
+ adapter_file.close();
+ }
+ } else {
+ LOG_ERR("Failed to save LoRA adapter\n");
+ }
+ } else {
+ LOG_ERR("No trained adapter available for saving\n");
+ }
+
+ llama_backend_free();
+
+ return 0;
+}
diff --git a/examples/training/finetune.cpp b/examples/training/finetune.cpp
index 416d8d8f6c8f..a7bd01e1575d 100644
--- a/examples/training/finetune.cpp
+++ b/examples/training/finetune.cpp
@@ -2,6 +2,7 @@
#include "common.h"
#include "log.h"
#include "llama.h"
+#include "ggml-backend.h"
#include
#include
@@ -13,6 +14,72 @@
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
+static bool training_supports_out_prod_f16(const common_params & params) {
+ std::vector devices;
+
+ if (!params.devices.empty()) {
+ devices.assign(params.devices.begin(), params.devices.end());
+ } else {
+ ggml_backend_dev_t gpu = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
+ if (gpu) {
+ devices.push_back(gpu);
+ }
+ }
+
+ if (devices.empty()) {
+ return true;
+ }
+
+ constexpr int64_t ne0 = 4;
+ constexpr int64_t ne1 = 3;
+ constexpr int64_t k = 2;
+
+ struct ggml_tensor src0 = {};
+ struct ggml_tensor src1 = {};
+ struct ggml_tensor dst = {};
+
+ src0.type = GGML_TYPE_F16;
+ src1.type = GGML_TYPE_F32;
+ dst.type = GGML_TYPE_F32;
+
+ src0.ne[0] = ne0; src0.ne[1] = k; src0.ne[2] = 1; src0.ne[3] = 1;
+ src1.ne[0] = ne1; src1.ne[1] = k; src1.ne[2] = 1; src1.ne[3] = 1;
+ dst.ne [0] = ne0; dst.ne [1] = ne1; dst.ne [2] = 1; dst.ne [3] = 1;
+
+ src0.nb[0] = sizeof(ggml_fp16_t);
+ src0.nb[1] = src0.nb[0] * ne0;
+ src0.nb[2] = src0.nb[1] * k;
+ src0.nb[3] = src0.nb[2] * 1;
+
+ src1.nb[0] = sizeof(float);
+ src1.nb[1] = src1.nb[0] * ne1;
+ src1.nb[2] = src1.nb[1] * k;
+ src1.nb[3] = src1.nb[2] * 1;
+
+ dst.nb[0] = sizeof(float);
+ dst.nb[1] = dst.nb[0] * ne0;
+ dst.nb[2] = dst.nb[1] * ne1;
+ dst.nb[3] = dst.nb[2] * 1;
+
+ dst.op = GGML_OP_OUT_PROD;
+ dst.src[0] = &src0;
+ dst.src[1] = &src1;
+
+ for (ggml_backend_dev_t dev : devices) {
+ if (dev == nullptr) {
+ continue;
+ }
+ if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
+ continue;
+ }
+ if (!ggml_backend_dev_supports_op(dev, &dst)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
int main(int argc, char ** argv) {
common_params params;
params.escape = false;
@@ -26,13 +93,16 @@ int main(int argc, char ** argv) {
__func__);
params.use_mmap = false;
}
- if (params.cache_type_k != GGML_TYPE_F32) {
- LOG_INF("%s: force changing k cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
- params.cache_type_k = GGML_TYPE_F32;
- }
- if (params.cache_type_v != GGML_TYPE_F32) {
- LOG_INF("%s: force changing v cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
- params.cache_type_v = GGML_TYPE_F32;
+ const bool supports_out_prod_f16 = training_supports_out_prod_f16(params);
+ if (!supports_out_prod_f16) {
+ if (params.cache_type_k != GGML_TYPE_F32) {
+ LOG_INF("%s: force changing k cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
+ params.cache_type_k = GGML_TYPE_F32;
+ }
+ if (params.cache_type_v != GGML_TYPE_F32) {
+ LOG_INF("%s: force changing v cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
+ params.cache_type_v = GGML_TYPE_F32;
+ }
}
common_init();
@@ -54,22 +124,18 @@ int main(int argc, char ** argv) {
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
}
- std::vector tokens = common_tokenize(ctx.get(), params.prompt, true);
- ggml_opt_dataset_t dataset = common_opt_dataset_init(ctx.get(), tokens, llama_n_ctx(ctx.get()) / 2);
+ std::vector tokens = common_tokenize(ctx.get(), params.prompt, true);
+ ggml_opt_dataset_t dataset = common_opt_dataset_init(ctx.get(), tokens, llama_n_ctx(ctx.get())/2);
struct lr_opt & lr = params.lr;
LOG_INF("-optimizer %s -lr0 %.2g -wd %.2g -lr-min %.2g -min-epochs %.2g -epochs %d -period %.2g -val %.2g\n",
ggml_opt_optimizer_name(params.optimizer), (double) lr.lr0, (double) lr.wd, (double) lr.lr_min, (double) lr.decay_epochs,
(unsigned) lr.epochs, (double) params.n_batch / params.n_ubatch, (double) params.val_split);
- struct llama_opt_params lopt_params{
- /*n_ctx_train =*/0,
- /*param_filter =*/llama_opt_param_filter_all,
- /*param_filter_ud =*/nullptr,
- /*get_opt_pars =*/common_opt_lr_pars,
- /*get_opt_pars_ud =*/¶ms.lr,
- /*optimizer_type =*/params.optimizer,
- };
+ struct llama_opt_params lopt_params = llama_opt_default_params();
+ lopt_params.get_opt_pars = common_opt_lr_pars;
+ lopt_params.get_opt_pars_ud = ¶ms.lr;
+ lopt_params.optimizer_type = params.optimizer;
llama_opt_init(ctx.get(), model.get(), lopt_params);
const int64_t idata_split = ggml_opt_dataset_ndata(dataset) * (1.0f - params.val_split);
@@ -79,7 +145,7 @@ int main(int argc, char ** argv) {
for (lr.epoch = 0; lr.epoch < lr.epochs; ++lr.epoch) {
llama_opt_epoch(ctx.get(), dataset, result_train, result_eval, idata_split,
- ggml_opt_epoch_callback_progress_bar, ggml_opt_epoch_callback_progress_bar);
+ ggml_opt_epoch_callback_progress_bar, ggml_opt_epoch_callback_progress_bar);
fprintf(stderr, "\n");
ggml_opt_result_reset(result_train);
diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt
index db47ae8dff2f..7d3723e7cf69 100644
--- a/ggml/CMakeLists.txt
+++ b/ggml/CMakeLists.txt
@@ -216,6 +216,7 @@ option(GGML_HIP_EXPORT_METRICS "ggml: enable kernel perf metrics ou
option(GGML_MUSA_GRAPHS "ggml: use MUSA graph, experimental, unstable" OFF)
option(GGML_MUSA_MUDNN_COPY "ggml: enable muDNN for accelerated copy" OFF)
option(GGML_VULKAN "ggml: use Vulkan" OFF)
+option(GGML_VULKAN_BUILD_ADRENO_SHADERS "ggml: build Adreno-supported shader variants" ON)
option(GGML_VULKAN_CHECK_RESULTS "ggml: run Vulkan op checks" OFF)
option(GGML_VULKAN_DEBUG "ggml: enable Vulkan debug output" OFF)
option(GGML_VULKAN_MEMORY_DEBUG "ggml: enable Vulkan memory debug output" OFF)
@@ -325,8 +326,15 @@ set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
#if (GGML_METAL)
# set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
#endif()
-install(TARGETS ggml LIBRARY PUBLIC_HEADER)
-install(TARGETS ggml-base LIBRARY)
+install(
+ TARGETS ggml ggml-base
+ EXPORT ggml-targets)
+
+install(
+ EXPORT ggml-targets
+ FILE ggml-targets.cmake
+ NAMESPACE ggml::
+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml)
if (GGML_STANDALONE)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in
@@ -377,7 +385,7 @@ set(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of
configure_package_config_file(
${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
- INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml
+ INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml
PATH_VARS GGML_INCLUDE_INSTALL_DIR
GGML_LIB_INSTALL_DIR
GGML_BIN_INSTALL_DIR)
@@ -396,8 +404,7 @@ message(STATUS "ggml commit: ${GGML_BUILD_COMMIT}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)
-
+ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/ggml)
if (MSVC)
set(MSVC_WARNING_FLAGS
/wd4005 # Macro redefinition
diff --git a/ggml/cmake/ggml-config.cmake.in b/ggml/cmake/ggml-config.cmake.in
index 91c9d5cd3434..97e44a000fc1 100644
--- a/ggml/cmake/ggml-config.cmake.in
+++ b/ggml/cmake/ggml-config.cmake.in
@@ -95,12 +95,22 @@ if (NOT GGML_SHARED_LIB)
list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)
endif()
endif()
+
+ if (GGML_OPENCL)
+ find_package(PkgConfig REQUIRED)
+ pkg_check_modules(OpenCL REQUIRED IMPORTED_TARGET OpenCL)
+ list(APPEND GGML_OPENCL_INTERFACE_LINK_LIBRARIES PkgConfig::OpenCL)
+ endif()
+
endif()
set_and_check(GGML_INCLUDE_DIR "@PACKAGE_GGML_INCLUDE_INSTALL_DIR@")
set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@")
#set_and_check(GGML_BIN_DIR "@PACKAGE_GGML_BIN_INSTALL_DIR@")
+# Include the exported targets file
+include("${CMAKE_CURRENT_LIST_DIR}/ggml-targets.cmake")
+
if(NOT TARGET ggml::ggml)
find_package(Threads REQUIRED)
diff --git a/ggml/include/ggml-opt.h b/ggml/include/ggml-opt.h
index 4703a05afe19..8ee2dc97b6e8 100644
--- a/ggml/include/ggml-opt.h
+++ b/ggml/include/ggml-opt.h
@@ -31,6 +31,7 @@ extern "C" {
GGML_OPT_LOSS_TYPE_MEAN,
GGML_OPT_LOSS_TYPE_SUM,
GGML_OPT_LOSS_TYPE_CROSS_ENTROPY,
+ GGML_OPT_LOSS_TYPE_CROSS_ENTROPY_MASKED,
GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR,
};
@@ -43,12 +44,23 @@ extern "C" {
int64_t ne_label, // number of elements per label
int64_t ndata, // total number of datapoints/labels
int64_t ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied)
+
+ GGML_API ggml_opt_dataset_t ggml_opt_dataset_init_with_masks(
+ enum ggml_type type_data, // the type for the internal data tensor
+ enum ggml_type type_label, // the type for the internal labels tensor
+ enum ggml_type type_mask, // the type for the internal masks tensor
+ int64_t ne_datapoint, // number of elements per datapoint
+ int64_t ne_label, // number of elements per label
+ int64_t ne_mask, // number of elements per mask
+ int64_t ndata, // total number of datapoints/labels
+ int64_t ndata_shard); // number of datapoints/labels per shard
GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset);
// get underlying tensors that store the data
GGML_API int64_t ggml_opt_dataset_ndata (ggml_opt_dataset_t dataset);
GGML_API struct ggml_tensor * ggml_opt_dataset_data (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata]
GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label, ndata]
+ GGML_API struct ggml_tensor * ggml_opt_dataset_masks (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata], can be null
// shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative
GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata);
@@ -65,6 +77,13 @@ extern "C" {
size_t nb_data_batch,
void * labels_batch,
int64_t ibatch);
+ GGML_API void ggml_opt_dataset_get_batch_host_with_masks(
+ ggml_opt_dataset_t dataset,
+ void * data_batch,
+ size_t nb_data_batch,
+ void * labels_batch,
+ void * masks_batch,
+ int64_t ibatch);
// ====== Model / Context ======
@@ -148,6 +167,7 @@ extern "C" {
GGML_API struct ggml_tensor * ggml_opt_inputs( ggml_opt_context_t opt_ctx); // forward graph input tensor
GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor
GGML_API struct ggml_tensor * ggml_opt_labels( ggml_opt_context_t opt_ctx); // labels to compare outputs against
+ GGML_API struct ggml_tensor * ggml_opt_masks( ggml_opt_context_t opt_ctx); // assistant masks for instruction tuning, can be null
GGML_API struct ggml_tensor * ggml_opt_loss( ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss
GGML_API struct ggml_tensor * ggml_opt_pred( ggml_opt_context_t opt_ctx); // predictions made by outputs
GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels
@@ -155,6 +175,19 @@ extern "C" {
// get the gradient accumulator for a node from the forward graph
GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node);
+ // get optimizer state tensors (momentum and variance for AdamW)
+ GGML_API int64_t ggml_opt_get_iter(ggml_opt_context_t opt_ctx);
+ GGML_API void ggml_opt_set_iter(ggml_opt_context_t opt_ctx, int64_t iter);
+ GGML_API int32_t ggml_opt_get_nparams(ggml_opt_context_t opt_ctx);
+ GGML_API struct ggml_tensor * ggml_opt_get_grad_m(ggml_opt_context_t opt_ctx, int32_t index);
+ GGML_API struct ggml_tensor * ggml_opt_get_grad_v(ggml_opt_context_t opt_ctx, int32_t index);
+
+ // ====== Optimizer State Persistence ======
+
+ GGML_API bool ggml_opt_save_state(ggml_opt_context_t opt_ctx, const char* filename);
+ GGML_API bool ggml_opt_load_state(ggml_opt_context_t opt_ctx, const char* filename);
+ GGML_API bool ggml_opt_load_tensors(ggml_opt_context_t opt_ctx, const char* filename);
+
GGML_API enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t); //TODO consistent naming scheme
GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type);
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 48da68fe7e3e..d8526feaa712 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -479,10 +479,12 @@ extern "C" {
GGML_OP_MEAN,
GGML_OP_ARGMAX,
GGML_OP_COUNT_EQUAL,
+ GGML_OP_COUNT_EQUAL_MASKED,
GGML_OP_REPEAT,
GGML_OP_REPEAT_BACK,
GGML_OP_CONCAT,
GGML_OP_SILU_BACK,
+ GGML_OP_GEGLU_BACK,
GGML_OP_NORM, // normalize
GGML_OP_RMS_NORM,
GGML_OP_RMS_NORM_BACK,
@@ -558,6 +560,8 @@ extern "C" {
GGML_OP_CROSS_ENTROPY_LOSS,
GGML_OP_CROSS_ENTROPY_LOSS_BACK,
+ GGML_OP_CROSS_ENTROPY_LOSS_MASKED,
+ GGML_OP_CROSS_ENTROPY_LOSS_MASKED_BACK,
GGML_OP_OPT_STEP_ADAMW,
GGML_OP_OPT_STEP_SGD,
@@ -1033,6 +1037,13 @@ extern "C" {
struct ggml_tensor * a,
struct ggml_tensor * b);
+ // count number of equal elements in a and b, but only where mask=1
+ GGML_API struct ggml_tensor * ggml_count_equal_masked(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // predictions
+ struct ggml_tensor * b, // targets
+ struct ggml_tensor * c); // mask (1 for positions to count, 0 to skip)
+
// if a is the same shape as b, and a is not parameter, return a
// otherwise, return a new tensor: repeat(a) to fit in b
GGML_API struct ggml_tensor * ggml_repeat(
@@ -1172,6 +1183,10 @@ extern "C" {
struct ggml_tensor * a,
struct ggml_tensor * b);
+ GGML_API struct ggml_tensor * ggml_geglu_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * grad,
+ struct ggml_tensor * g);
// hardswish(x) = x * relu6(x + 3) / 6
GGML_API struct ggml_tensor * ggml_hardswish(
struct ggml_context * ctx,
@@ -2530,6 +2545,19 @@ extern "C" {
struct ggml_tensor * b, // labels
struct ggml_tensor * c); // gradients of cross_entropy_loss result
+ // Masked cross-entropy loss for instruction fine-tuning (assistant-only loss)
+ GGML_API struct ggml_tensor * ggml_cross_entropy_loss_masked(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // logits
+ struct ggml_tensor * b, // labels
+ struct ggml_tensor * c); // mask (1 for assistant tokens, 0 for masked)
+ GGML_API struct ggml_tensor * ggml_cross_entropy_loss_masked_back(
+ struct ggml_context * ctx,
+ struct ggml_tensor * a, // logits
+ struct ggml_tensor * b, // labels
+ struct ggml_tensor * c, // mask
+ struct ggml_tensor * d); // gradients of cross_entropy_loss result
+
// AdamW optimizer step
// Paper: https://arxiv.org/pdf/1711.05101v3.pdf
// PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html
diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h
index 79ee202062b0..db16883c0536 100644
--- a/ggml/include/gguf.h
+++ b/ggml/include/gguf.h
@@ -74,11 +74,18 @@ extern "C" {
// if not NULL, create a ggml_context and allocate the tensor data in it
struct ggml_context ** ctx;
+
+ // if true, stop parsing immediately after the KV pairs and skip tensor info entirely;
+ // ctx is ignored when this flag is set
+#ifdef __cplusplus
+ bool kv_only = false;
+#else
+ bool kv_only;
+#endif
};
GGML_API struct gguf_context * gguf_init_empty(void);
GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
- //GGML_API struct gguf_context * gguf_init_from_buffer(..);
GGML_API void gguf_free(struct gguf_context * ctx);
@@ -200,3 +207,8 @@ extern "C" {
#ifdef __cplusplus
}
#endif
+
+#if defined(__cplusplus)
+#include
+GGML_API struct gguf_context * gguf_init_from_buffer(std::basic_streambuf& streambuf, struct gguf_init_params params);
+#endif
diff --git a/ggml/include/uint8-buff-stream.h b/ggml/include/uint8-buff-stream.h
new file mode 100644
index 000000000000..e372f917b68e
--- /dev/null
+++ b/ggml/include/uint8-buff-stream.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+
+#ifdef GGML_SHARED
+# if defined(_WIN32) && !defined(__MINGW32__)
+# ifdef GGML_BUILD
+# define GGML_CLASS_API __declspec(dllexport)
+# else
+# define GGML_CLASS_API __declspec(dllimport)
+# endif
+# else
+# define GGML_CLASS_API __attribute__((visibility("default")))
+# endif
+#else
+# define GGML_CLASS_API
+#endif
+
+/// @brief Custom basic_streambuf for uint8_t input data, that owns the underlying data.
+/// @note basic_streambuf has more support on different platforms than basic_streambuf
+/// which is missing on some platforms (e.g. MacOS, newer NDKs). C++ 17 provides additional guarantees for char.
+class GGML_CLASS_API Uint8BufferStreamBuf : public std::basic_streambuf {
+ public:
+ Uint8BufferStreamBuf(std::vector && _data);
+
+ protected:
+ int_type underflow() override;
+
+ /// @brief Efficient bulk reading. The standard implementation specifies that this function can be overridden
+ /// to provide a more efficient implementation: sgetn will call this function if it is overridden.
+ std::streamsize xsgetn(char_type * s, std::streamsize n) override;
+
+ pos_type seekoff(off_type off, std::ios_base::seekdir dir,
+ std::ios_base::openmode which = std::ios_base::in) override;
+
+ pos_type seekpos(pos_type pos, std::ios_base::openmode which = std::ios_base::in) override;
+
+ private:
+ std::vector data;
+};
diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt
index d93664b8b58b..f0aee7765a14 100644
--- a/ggml/src/CMakeLists.txt
+++ b/ggml/src/CMakeLists.txt
@@ -189,10 +189,6 @@ endif()
# ggml
-if (GGML_BACKEND_DL AND NOT BUILD_SHARED_LIBS)
- message(FATAL_ERROR "GGML_BACKEND_DL requires BUILD_SHARED_LIBS")
-endif()
-
add_library(ggml-base
../include/ggml.h
../include/ggml-alloc.h
@@ -200,6 +196,7 @@ add_library(ggml-base
../include/ggml-cpp.h
../include/ggml-opt.h
../include/gguf.h
+ ../include/uint8-buff-stream.h
ggml.c
ggml.cpp
ggml-alloc.c
@@ -209,7 +206,8 @@ add_library(ggml-base
ggml-threading.h
ggml-quants.c
ggml-quants.h
- gguf.cpp)
+ gguf.cpp
+ uint8-buff-stream.cpp)
set_target_properties(ggml-base PROPERTIES
VERSION ${GGML_VERSION}
@@ -251,24 +249,37 @@ function(ggml_add_backend_library backend)
if (GGML_BACKEND_DL)
add_library(${backend} MODULE ${ARGN})
# write the shared library to the output directory
- set_target_properties(${backend} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+ # Set output name with qvac- prefix to match backend_filename_prefix() in ggml-backend-reg.cpp
+ set_target_properties(${backend} PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
+ OUTPUT_NAME "qvac-${backend}")
target_compile_definitions(${backend} PRIVATE GGML_BACKEND_DL)
- add_dependencies(ggml ${backend})
+ # Do not add dependency, the User will have to explicitely build and install
+ # the available `ggml::ggml-*` backend targets. This is for better integration
+ # with cmake-bare
+ # add_dependencies(ggml ${backend})
+
if (GGML_BACKEND_DIR)
- install(TARGETS ${backend} LIBRARY DESTINATION ${GGML_BACKEND_DIR})
+ install(TARGETS ${backend}
+ EXPORT ggml-targets
+ LIBRARY DESTINATION ${GGML_BACKEND_DIR}
+ RUNTIME DESTINATION ${GGML_BACKEND_DIR})
else()
- install(TARGETS ${backend} LIBRARY DESTINATION ${CMAKE_INSTALL_BINDIR})
+ install(TARGETS ${backend}
+ EXPORT ggml-targets
+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
endif()
else()
add_library(${backend} ${ARGN})
target_link_libraries(ggml PUBLIC ${backend})
- install(TARGETS ${backend} LIBRARY)
+ install(TARGETS ${backend} EXPORT ggml-targets)
endif()
target_link_libraries(${backend} PRIVATE ggml-base)
target_include_directories(${backend} PRIVATE ..)
- if (${BUILD_SHARED_LIBS})
+ if (${BUILD_SHARED_LIBS} OR GGML_BACKEND_DL)
target_compile_definitions(${backend} PRIVATE GGML_BACKEND_BUILD)
target_compile_definitions(${backend} PUBLIC GGML_BACKEND_SHARED)
endif()
@@ -467,7 +478,7 @@ if(CMAKE_SYSTEM_NAME MATCHES "visionOS")
target_compile_definitions(ggml-base PUBLIC _DARWIN_C_SOURCE)
endif()
-if (BUILD_SHARED_LIBS)
+if (BUILD_SHARED_LIBS OR GGML_BACKEND_DL)
foreach (target ggml-base ggml)
set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON)
target_compile_definitions(${target} PRIVATE GGML_BUILD)
diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp
index e96b5c403dd3..bbe8983243b3 100644
--- a/ggml/src/ggml-backend-reg.cpp
+++ b/ggml/src/ggml-backend-reg.cpp
@@ -4,11 +4,13 @@
#include
#include
#include
+#include
#include
#include
#include
#include
#include
+#include
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
@@ -498,9 +500,9 @@ static fs::path get_executable_path() {
static fs::path backend_filename_prefix() {
#ifdef _WIN32
- return fs::u8path("ggml-");
+ return fs::u8path("qvac-ggml-");
#else
- return fs::u8path("libggml-");
+ return fs::u8path("libqvac-ggml-");
#endif
}
@@ -515,7 +517,7 @@ static fs::path backend_filename_extension() {
static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {
// enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
const fs::path name_path = fs::u8path(name);
- const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path("-").native();
+ const fs::path file_prefix = backend_filename_prefix().native() + name_path.native();
const fs::path file_extension = backend_filename_extension();
std::vector search_paths;
@@ -526,6 +528,9 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
// default search paths: executable directory, current directory
search_paths.push_back(get_executable_path());
search_paths.push_back(fs::current_path());
+
+ // Android does not require prepending path, the .apk will have embedded the dynamic .so, only the name is needed for dlopen
+ // TODO add here prebuild/ search patch for Desktop platforms where we want to support dynamic loading
} else {
search_paths.push_back(fs::u8path(user_search_path));
}
@@ -533,38 +538,41 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
int best_score = 0;
fs::path best_path;
+ auto tryEntryWithScore = [&best_score, &best_path, silent, _func = __func__](const fs::path & entryPath,
+ int scoreOffset = 1) {
+ dl_handle_ptr handle{ dl_load_library(entryPath) };
+ if (!handle && !silent) {
+ GGML_LOG_ERROR("%s: failed to load %s: %s\n", _func, path_str(entryPath).c_str(), dl_error());
+ }
+ if (handle) {
+ auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
+ int s = 1;
+ if (score_fn) {
+ s = score_fn() + scoreOffset;
+ }
+#ifdef NDEBUG
+ GGML_LOG_DEBUG("%s: %s score: %d\n", _func, path_str(entryPath).c_str(), s);
+#endif
+ if (s > best_score) {
+ best_score = s;
+ best_path = entryPath;
+ }
+ }
+ };
+
for (const auto & search_path : search_paths) {
if (!fs::exists(search_path)) {
GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
continue;
}
+ GGML_LOG_INFO("%s: searching for %s in %s\n", __func__, path_str(name_path).c_str(), path_str(search_path).c_str());
fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);
for (const auto & entry : dir_it) {
if (entry.is_regular_file()) {
auto filename = entry.path().filename();
auto ext = entry.path().extension();
if (filename.native().find(file_prefix) == 0 && ext == file_extension) {
- dl_handle_ptr handle { dl_load_library(entry) };
- if (!handle && !silent) {
- GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path_str(entry.path()).c_str(), dl_error());
- }
- if (handle) {
- auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");
- if (score_fn) {
- int s = score_fn();
-#ifndef NDEBUG
- GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, path_str(entry.path()).c_str(), s);
-#endif
- if (s > best_score) {
- best_score = s;
- best_path = entry.path();
- }
- } else {
- if (!silent) {
- GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, path_str(entry.path()).c_str());
- }
- }
- }
+ tryEntryWithScore(entry.path());
}
}
}
@@ -579,7 +587,27 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
return get_reg().load_backend(path, silent);
}
}
- return nullptr;
+ }
+
+ // In the case of Android, we can load with just the library filename, without pre-pending any path
+ if (best_path.empty()) {
+ // From worst to best
+ std::vector names = { name_path };
+#ifdef __ANDROID__
+ if (strcmp(name, "cpu") == 0) {
+ names.emplace_back("cpu-android_armv8.0_1");
+ names.emplace_back("cpu-android_armv8.2_1");
+ names.emplace_back("cpu-android_armv8.2_2");
+ names.emplace_back("cpu-android_armv8.6_1");
+ }
+#endif
+ for (size_t scoreOffset = 0; scoreOffset < names.size(); ++scoreOffset) {
+ const auto & loopNamePath = names[scoreOffset];
+ // Try loading backend with just the library name, leave to dlopen path resolution.
+ fs::path filename = backend_filename_prefix().native() + loopNamePath.native() +
+ backend_filename_extension().native();
+ tryEntryWithScore(filename, 1+scoreOffset);
+ }
}
return get_reg().load_backend(best_path, silent);
@@ -589,7 +617,54 @@ void ggml_backend_load_all() {
ggml_backend_load_all_from_path(nullptr);
}
+#ifdef __ANDROID__
+namespace {
+// Parses adreno version from gpu description or returns -1 if its not Adreno GPU or -3 if failed to parse the version
+int adrenoVersion(const std::string & gpuDescription) {
+ std::regex adrenoRegex(R"((\d+))");
+ std::smatch matches;
+ if (gpuDescription.find("dreno") != std::string::npos && std::regex_search(gpuDescription, matches, adrenoRegex) && matches.size() > 1) {
+ try {
+ int adrenoVersion = std::stoi(matches[1].str());
+ return adrenoVersion;
+ } catch (std::invalid_argument & e) {
+ GGML_LOG_ERROR("%s: failed to parse adreno version from %s: %s\n", __func__, gpuDescription.c_str(),
+ e.what());
+ return -3;
+ }
+ }
+ return -1;
+}
+
+// Returns smallest Adreno version among GPU devices or -1 if there is no adreno GPU
+int minAdrenoVersion(ggml_backend_reg_t vulkanBackend) {
+ if (!vulkanBackend) {
+ return -2;
+ }
+ int minFoundVersion = std::numeric_limits::max();
+ for (size_t i = 0; i < vulkanBackend->iface.get_device_count(vulkanBackend); i++) {
+ ggml_backend_dev_t dev = vulkanBackend->iface.get_device(vulkanBackend, i);
+ if (!dev) {
+ continue;
+ }
+ auto description = std::string(dev->iface.get_description(dev));
+ GGML_LOG_INFO("%s: found device description: %s\n", __func__, description.c_str());
+ int devAdrenoVersion = adrenoVersion(description);
+ if (devAdrenoVersion > 0) {
+ minFoundVersion = std::min(minFoundVersion, devAdrenoVersion);
+ }
+ }
+ if (minFoundVersion < std::numeric_limits::max()) {
+ return minFoundVersion;
+ }
+ return -1;
+}
+} // namespace
+#endif
+
void ggml_backend_load_all_from_path(const char * dir_path) {
+#ifdef GGML_BACKEND_DL
+ // Only attempt to dlopen backends when built with dynamic backend support
#ifdef NDEBUG
bool silent = true;
#else
@@ -604,7 +679,34 @@ void ggml_backend_load_all_from_path(const char * dir_path) {
ggml_backend_load_best("rpc", silent, dir_path);
ggml_backend_load_best("sycl", silent, dir_path);
ggml_backend_load_best("vulkan", silent, dir_path);
- ggml_backend_load_best("opencl", silent, dir_path);
+
+ bool useOpencl = true;
+
+#ifdef __ANDROID__
+ // Logic for buggy backends on Adreno GPUs
+ // Use Vulkan backend to obtain GPU information
+ ggml_backend_reg_t vulkanBackend = ggml_backend_reg_by_name("vulkan");
+ int devicesMinAdrenoVersion = minAdrenoVersion(vulkanBackend);
+ if (devicesMinAdrenoVersion <= 0) {
+ GGML_LOG_INFO(
+ "%s: no adreno GPU version found (%d) removing OpenCL backend (if any) to rely on Vulkan/cpu only\n",
+ __func__, devicesMinAdrenoVersion);
+ useOpencl = false;
+ } else if (devicesMinAdrenoVersion > 700) {
+ GGML_LOG_INFO("%s: Adreno GPU version %d found keeping OpenCL backend\n", __func__, devicesMinAdrenoVersion);
+ } else if (devicesMinAdrenoVersion > 600) {
+ GGML_LOG_INFO("%s: Adreno GPU version %d should rely on cpu only\n", __func__, devicesMinAdrenoVersion);
+ if (vulkanBackend) {
+ ggml_backend_unload(vulkanBackend);
+ GGML_LOG_INFO("%s: Vulkan backend removed\n", __func__);
+ }
+ useOpencl = false;
+ }
+#endif
+
+ if(useOpencl) {
+ ggml_backend_load_best("opencl", silent, dir_path);
+ }
ggml_backend_load_best("hexagon", silent, dir_path);
ggml_backend_load_best("musa", silent, dir_path);
ggml_backend_load_best("cpu", silent, dir_path);
@@ -613,4 +715,9 @@ void ggml_backend_load_all_from_path(const char * dir_path) {
if (backend_path) {
ggml_backend_load(backend_path);
}
+#else
+ // When built without GGML_BACKEND_DL, backends are statically linked
+ // No dynamic loading needed - avoids potential conflicts with system libraries
+ GGML_UNUSED(dir_path);
+#endif
}
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index 08681f35e3f9..29b8281f3e9a 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -1,5 +1,6 @@
// Note: porting this file to C++ is a work in progress
+#include
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
@@ -1634,7 +1635,11 @@ ggml_backend_sched_t ggml_backend_sched_new(
struct ggml_backend_sched * sched = (ggml_backend_sched *) calloc(1, sizeof(struct ggml_backend_sched));
+#ifndef FORCE_GGML_VK_PERF_LOGGER
const char * GGML_SCHED_DEBUG = getenv("GGML_SCHED_DEBUG");
+#else
+ GGML_SCHED_DEBUG = "2";
+#endif
sched->debug = GGML_SCHED_DEBUG ? atoi(GGML_SCHED_DEBUG) : 0;
sched->debug_realloc = 0;
diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h
index 0775c87f98b3..d72e14244a8c 100644
--- a/ggml/src/ggml-cpu/arch-fallback.h
+++ b/ggml/src/ggml-cpu/arch-fallback.h
@@ -13,9 +13,12 @@
#define ggml_vec_dot_q5_0_q8_0_generic ggml_vec_dot_q5_0_q8_0
#define ggml_vec_dot_q5_1_q8_1_generic ggml_vec_dot_q5_1_q8_1
#define ggml_vec_dot_q8_0_q8_0_generic ggml_vec_dot_q8_0_q8_0
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_0_generic ggml_vec_dot_tq2_0_q8_0
+#define ggml_vec_dot_tq2_0_q8_1_generic ggml_vec_dot_tq2_0_q8_1
#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
#define ggml_vec_dot_q3_K_q8_K_generic ggml_vec_dot_q3_K_q8_K
#define ggml_vec_dot_q4_K_q8_K_generic ggml_vec_dot_q4_K_q8_K
@@ -59,6 +62,7 @@
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
// repack.cpp
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
@@ -71,13 +75,17 @@
#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0
#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
#elif defined(__POWERPC__) || defined(__powerpc__)
// ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679
// quants.c
#define quantize_row_q8_K_generic quantize_row_q8_K
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_0_generic ggml_vec_dot_tq2_0_q8_0
+#define ggml_vec_dot_tq2_0_q8_1_generic ggml_vec_dot_tq2_0_q8_1
#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
// repack.cpp
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
@@ -104,8 +112,11 @@
#define quantize_row_q8_K_generic quantize_row_q8_K
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_0_generic ggml_vec_dot_tq2_0_q8_0
+#define ggml_vec_dot_tq2_0_q8_1_generic ggml_vec_dot_tq2_0_q8_1
#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
// repack.cpp
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
@@ -132,6 +143,8 @@
#define quantize_row_q8_K_generic quantize_row_q8_K
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_0_generic ggml_vec_dot_tq2_0_q8_0
+#define ggml_vec_dot_tq2_0_q8_1_generic ggml_vec_dot_tq2_0_q8_1
#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K
@@ -142,6 +155,7 @@
#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0
#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K
#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
// repack.cpp
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
@@ -166,6 +180,7 @@
#define quantize_row_q8_K_generic quantize_row_q8_K
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_1_generic ggml_vec_dot_tq2_0_q8_1
#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
@@ -174,6 +189,8 @@
#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K
#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K
#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
+#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
// repack.cpp
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
@@ -200,6 +217,7 @@
#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
+#define ggml_vec_dot_tq2_0_q8_1_generic ggml_vec_dot_tq2_0_q8_1
#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K
@@ -210,6 +228,7 @@
#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0
#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K
#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
+#define ggml_vec_dot_q8_1_q8_1_generic ggml_vec_dot_q8_1_q8_1
// repack.cpp
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c
index b390ab61c785..72a7840bc27a 100644
--- a/ggml/src/ggml-cpu/arch/arm/quants.c
+++ b/ggml/src/ggml-cpu/arch/arm/quants.c
@@ -1415,6 +1415,14 @@ void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
#endif
}
+void ggml_vec_dot_tq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ ggml_vec_dot_tq2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_tq2_0_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ ggml_vec_dot_tq2_0_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
+}
+
void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
assert(nrc == 1);
UNUSED(nrc);
diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c
index cb49320a67f1..20c7c3726674 100644
--- a/ggml/src/ggml-cpu/arch/x86/quants.c
+++ b/ggml/src/ggml-cpu/arch/x86/quants.c
@@ -1274,6 +1274,14 @@ void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
#endif
}
+void ggml_vec_dot_tq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ ggml_vec_dot_tq2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
+}
+
+void ggml_vec_dot_tq2_0_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ ggml_vec_dot_tq2_0_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
+}
+
void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
assert(nrc == 1);
UNUSED(nrc);
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 8507557267a3..04ce9d8c37be 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -250,6 +250,7 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
},
[GGML_TYPE_Q8_1] = {
.from_float = quantize_row_q8_1,
+ .vec_dot = ggml_vec_dot_q8_1_q8_1,
.vec_dot_type = GGML_TYPE_Q8_1,
.nrows = 1,
},
@@ -382,6 +383,33 @@ const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type
return &type_traits_cpu[type];
}
+struct vec_dot_selection {
+ ggml_vec_dot_t vec_dot;
+ enum ggml_type vec_dot_type;
+};
+
+static struct vec_dot_selection ggml_cpu_select_vec_dot(
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * src1) {
+
+ struct vec_dot_selection selection = {
+ .vec_dot = type_traits_cpu[src0->type].vec_dot,
+ .vec_dot_type = type_traits_cpu[src0->type].vec_dot_type,
+ };
+
+ if (src0->type == GGML_TYPE_TQ2_0 && src1 != NULL) {
+ if (src1->type == GGML_TYPE_Q8_1) {
+ selection.vec_dot = ggml_vec_dot_tq2_0_q8_1;
+ selection.vec_dot_type = GGML_TYPE_Q8_1;
+ } else if (src1->type == GGML_TYPE_Q8_0) {
+ selection.vec_dot = ggml_vec_dot_tq2_0_q8_0;
+ selection.vec_dot_type = GGML_TYPE_Q8_0;
+ }
+ }
+
+ return selection;
+}
+
//
// Threading defs
//
@@ -1112,7 +1140,8 @@ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,
static void ggml_compute_forward_mul_mat_one_chunk(
const struct ggml_compute_params * params,
struct ggml_tensor * dst,
- const enum ggml_type type,
+ ggml_vec_dot_t vec_dot,
+ enum ggml_type vec_dot_type,
const int64_t num_rows_per_vec_dot,
const int64_t ir0_start,
const int64_t ir0_end,
@@ -1125,10 +1154,6 @@ static void ggml_compute_forward_mul_mat_one_chunk(
GGML_TENSOR_BINARY_OP_LOCALS
const bool src1_cont = ggml_is_contiguous(src1);
-
- ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
- enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
-
// broadcast factors
const int64_t r2 = ne12 / ne02;
const int64_t r3 = ne13 / ne03;
@@ -1211,9 +1236,11 @@ void ggml_compute_forward_mul_mat(
const int ith = params->ith;
const int nth = params->nth;
- enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
- ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
- int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;
+ struct vec_dot_selection vds = ggml_cpu_select_vec_dot(src0, src1);
+ ggml_vec_dot_t vec_dot = vds.vec_dot;
+ enum ggml_type vec_dot_type = vds.vec_dot_type;
+ ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
+ int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;
GGML_ASSERT(ne0 == ne01);
GGML_ASSERT(ne1 == ne11);
@@ -1383,7 +1410,7 @@ UseGgmlGemm2:;
if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) {
num_rows_per_vec_dot = 1;
}
- ggml_compute_forward_mul_mat_one_chunk(params, dst, src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);
+ ggml_compute_forward_mul_mat_one_chunk(params, dst, vec_dot, vec_dot_type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);
if (nth >= nchunk0 * nchunk1) {
break;
@@ -1406,6 +1433,8 @@ static void ggml_compute_forward_mul_mat_id_one_chunk(
struct ggml_tensor * dst,
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
+ ggml_vec_dot_t vec_dot,
+ enum ggml_type vec_dot_type,
const struct ggml_tensor * ids,
const int64_t cur_a,
const int64_t ir0_start,
@@ -1420,11 +1449,6 @@ static void ggml_compute_forward_mul_mat_id_one_chunk(
GGML_TENSOR_BINARY_OP_LOCALS
- const enum ggml_type type = src0->type;
-
- ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
- enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
-
const int64_t blck_0 = 16;
const int64_t blck_1 = 16;
@@ -1490,7 +1514,13 @@ static void ggml_compute_forward_mul_mat_id(
const bool src1_cont = ggml_is_contiguous(src1);
- enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
+ ggml_vec_dot_t vec_dot;
+ enum ggml_type vec_dot_type;
+ {
+ struct vec_dot_selection vds = ggml_cpu_select_vec_dot(src0, src1);
+ vec_dot = vds.vec_dot;
+ vec_dot_type = vds.vec_dot_type;
+ }
ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
// we don't support permuted src0 or src1
@@ -1634,7 +1664,7 @@ static void ggml_compute_forward_mul_mat_id(
const int64_t ir1_end = MIN(ir1_start + dr1, nr1);
ggml_compute_forward_mul_mat_id_one_chunk(
- dst, src0, src1, ids, cur_a,
+ dst, src0, src1, vec_dot, vec_dot_type, ids, cur_a,
ir0_start, ir0_end, ir1_start, ir1_end,
src0_cur, matrix_rows, row_size, src1_cont, wdata
);
@@ -1739,6 +1769,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
{
ggml_compute_forward_count_equal(params, tensor);
} break;
+ case GGML_OP_COUNT_EQUAL_MASKED:
+ {
+ ggml_compute_forward_count_equal_masked(params, tensor);
+ } break;
case GGML_OP_REPEAT:
{
ggml_compute_forward_repeat(params, tensor);
@@ -1755,6 +1789,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
{
ggml_compute_forward_silu_back(params, tensor);
} break;
+ case GGML_OP_GEGLU_BACK:
+ {
+ ggml_compute_forward_geglu_back(params, tensor);
+ } break;
case GGML_OP_NORM:
{
ggml_compute_forward_norm(params, tensor);
@@ -2024,6 +2062,16 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
ggml_compute_forward_cross_entropy_loss_back(params, tensor);
}
break;
+ case GGML_OP_CROSS_ENTROPY_LOSS_MASKED:
+ {
+ ggml_compute_forward_cross_entropy_loss_masked(params, tensor);
+ }
+ break;
+ case GGML_OP_CROSS_ENTROPY_LOSS_MASKED_BACK:
+ {
+ ggml_compute_forward_cross_entropy_loss_masked_back(params, tensor);
+ }
+ break;
case GGML_OP_OPT_STEP_ADAMW:
{
ggml_compute_forward_opt_step_adamw(params, tensor);
@@ -2173,6 +2221,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
} break;
case GGML_OP_COUNT_EQUAL:
case GGML_OP_SOLVE_TRI:
+ case GGML_OP_COUNT_EQUAL_MASKED:
{
n_tasks = n_threads;
} break;
@@ -2233,6 +2282,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
}
break;
case GGML_OP_SILU_BACK:
+ case GGML_OP_GEGLU_BACK:
case GGML_OP_MUL:
case GGML_OP_DIV:
case GGML_OP_NORM:
@@ -2366,6 +2416,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
} break;
case GGML_OP_CROSS_ENTROPY_LOSS:
case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
+ case GGML_OP_CROSS_ENTROPY_LOSS_MASKED:
+ case GGML_OP_CROSS_ENTROPY_LOSS_MASKED_BACK:
case GGML_OP_OPT_STEP_ADAMW:
case GGML_OP_OPT_STEP_SGD:
{
@@ -2750,12 +2802,14 @@ struct ggml_cplan ggml_graph_plan(
}
} break;
case GGML_OP_COUNT_EQUAL:
+ case GGML_OP_COUNT_EQUAL_MASKED:
{
cur = ggml_type_size(node->type)*n_tasks;
} break;
case GGML_OP_MUL_MAT:
{
- const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;
+ struct vec_dot_selection vds = ggml_cpu_select_vec_dot(node->src[0], node->src[1]);
+ enum ggml_type vec_dot_type = vds.vec_dot_type;
if (node->src[1]->type != vec_dot_type) {
cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));
@@ -2767,7 +2821,8 @@ struct ggml_cplan ggml_graph_plan(
const struct ggml_tensor * src0 = node->src[0];
const struct ggml_tensor * src1 = node->src[1];
const struct ggml_tensor * ids = node->src[2];
- const enum ggml_type vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
+ struct vec_dot_selection vds = ggml_cpu_select_vec_dot(src0, src1);
+ enum ggml_type vec_dot_type = vds.vec_dot_type;
const int n_as = src0->ne[2];
// src1
if (src1->type != vec_dot_type) {
@@ -2868,6 +2923,11 @@ struct ggml_cplan ggml_graph_plan(
{
cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);
} break;
+ case GGML_OP_CROSS_ENTROPY_LOSS_MASKED:
+ case GGML_OP_CROSS_ENTROPY_LOSS_MASKED_BACK:
+ {
+ cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks) + sizeof(int64_t)*n_tasks;
+ } break;
case GGML_OP_COUNT:
{
GGML_ABORT("fatal error");
diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
index 3191faaa4cd9..eb7d4e738c9d 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.cpp
+++ b/ggml/src/ggml-cpu/ggml-cpu.cpp
@@ -436,7 +436,11 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
op->type != GGML_TYPE_IQ1_S &&
op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float
case GGML_OP_MUL_MAT:
- return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type;
+ return
+ src1->type == GGML_TYPE_F32 ||
+ src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type ||
+ (src0->type == GGML_TYPE_TQ2_0 &&
+ (src1->type == GGML_TYPE_Q8_1 || src1->type == GGML_TYPE_Q8_0));
case GGML_OP_SOFT_MAX_BACK: {
if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) {
return false;
@@ -452,7 +456,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
case GGML_OP_GET_ROWS_BACK:
return src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16;
case GGML_OP_OUT_PROD:
- return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&
+ return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&
src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
default:
return true;
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index 608e82af69f4..96949f2bfd20 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -1684,6 +1684,100 @@ void ggml_compute_forward_count_equal(
}
}
+// ggml_compute_forward_count_equal_masked
+
+static void ggml_compute_forward_count_equal_masked_i32(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const ggml_tensor * src2 = dst->src[2];
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ GGML_ASSERT(src0->type == GGML_TYPE_I32);
+ GGML_ASSERT(src1->type == GGML_TYPE_I32);
+ GGML_ASSERT(src2->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_are_same_shape(src0, src1));
+ GGML_ASSERT(src2->ne[1] == src0->ne[0] || ggml_are_same_shape(src0, src2));
+ GGML_ASSERT(ggml_is_scalar(dst));
+ GGML_ASSERT(dst->type == GGML_TYPE_I64);
+
+ const int64_t nr = ggml_nrows(src0);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ int64_t * sums = (int64_t *) params->wdata;
+ int64_t sum_thread = 0;
+
+ const int64_t dr = (nr + nth - 1)/nth;
+
+ const int64_t ir0 = dr*ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
+
+ for (int64_t ir = ir0; ir < ir1; ++ir) {
+ const int64_t i03 = ir / (ne02*ne01);
+ const int64_t i02 = (ir - i03*ne03) / ne01;
+ const int64_t i01 = ir - i03*ne03 - i02*ne02;
+
+ const char * data0 = (const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01;
+ const char * data1 = (const char *) src1->data + i03*nb13 + i02*nb12 + i01*nb11;
+ const char * data2 = (const char *) src2->data + i03*src2->nb[3] + i02*src2->nb[2] + i01*src2->nb[1];
+
+ for (int64_t i00 = 0; i00 < ne00; ++i00) {
+ const int32_t val0 = *((const int32_t *) (data0 + i00*nb00));
+ const int32_t val1 = *((const int32_t *) (data1 + i00*nb10));
+
+ float mask_val;
+ if (ggml_are_same_shape(src0, src2)) {
+ mask_val = *((const float *) (data2 + i00*src2->nb[0]));
+ } else {
+ const char * mask_ptr = (const char *) src2->data + 0*src2->nb[0] + i00*src2->nb[1] + 0*src2->nb[2];
+ mask_val = *((const float *) mask_ptr);
+ }
+
+ const bool mask = mask_val > 0.5f;
+
+ if (mask == 1) {
+ sum_thread += val0 == val1;
+ }
+ }
+ }
+ if (ith != 0) {
+ sums[ith] = sum_thread;
+ }
+ ggml_barrier(params->threadpool);
+
+ if (ith != 0) {
+ return;
+ }
+
+ for (int ith_other = 1; ith_other < nth; ++ith_other) {
+ sum_thread += sums[ith_other];
+ }
+ *((int64_t *) dst->data) = sum_thread;
+}
+
+void ggml_compute_forward_count_equal_masked(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_I32:
+ {
+ ggml_compute_forward_count_equal_masked_i32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
// ggml_compute_forward_repeat
static void ggml_compute_forward_repeat_f32(
@@ -2772,6 +2866,57 @@ void ggml_compute_forward_silu_back(
}
}
+static void ggml_compute_forward_geglu_back_f32(
+ const ggml_compute_params * params,
+ const struct ggml_tensor * grad,
+ const struct ggml_tensor * g,
+ struct ggml_tensor * dst) {
+
+ GGML_ASSERT(ggml_can_repeat(grad, dst));
+ GGML_ASSERT(ggml_are_same_shape(g, dst));
+ GGML_ASSERT(grad->type == GGML_TYPE_F32);
+ GGML_ASSERT(g->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nc = dst->ne[0];
+
+ const size_t nb1 = dst->nb[1];
+ const size_t nb2 = dst->nb[2];
+ const size_t nb3 = dst->nb[3];
+
+ for (int i3 = 0; i3 < dst->ne[3]; i3++) {
+ for (int i2 = 0; i2 < dst->ne[2]; i2++) {
+ for (int i1 = ith; i1 < dst->ne[1]; i1 += nth) {
+ float * dst_ptr = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ const float * grad_ptr = (const float *)((char *) grad->data + i3*grad->nb[3] + i2*grad->nb[2] + i1*grad->nb[1]);
+ const float * g_ptr = (const float *)((char *) g->data + i3*g->nb[3] + i2*g->nb[2] + i1*g->nb[1]);
+ ggml_vec_gelu_backward_f32(nc, dst_ptr, g_ptr, grad_ptr);
+ }
+ }
+ }
+}
+
+void ggml_compute_forward_geglu_back(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const struct ggml_tensor * grad = dst->src[0];
+ const struct ggml_tensor * g = dst->src[1];
+
+ switch (dst->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_geglu_back_f32(params, grad, g, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
// ggml_compute_forward_reglu
static void ggml_compute_forward_reglu_f32(
@@ -4168,6 +4313,107 @@ static void ggml_compute_forward_out_prod_f32(
}
}
+static void ggml_compute_forward_out_prod_f16_f32(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ GGML_ASSERT(ne0 == ne00);
+ GGML_ASSERT(ne1 == ne10);
+ GGML_ASSERT(ne2 == ne12);
+ GGML_ASSERT(ne3 == ne13);
+
+ GGML_ASSERT(ne2 % ne02 == 0);
+ GGML_ASSERT(ne3 % ne03 == 0);
+
+ // we don't support permuted src0 or src1
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ // GGML_ASSERT(nb0 <= nb1);
+ // GGML_ASSERT(nb1 <= nb2);
+ // GGML_ASSERT(nb2 <= nb3);
+
+ // nb01 >= nb00 - src0 is not transposed
+ // compute by src0 rows
+
+ if (ith == 0) {
+ ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);
+ }
+ ggml_barrier(params->threadpool);
+
+ // dst[:,:,:,:] = 0
+ // for i2,i3:
+ // for i1:
+ // for i01:
+ // for i0:
+ // dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]
+
+ // parallelize by last three dimensions
+
+ // total rows in dst
+ const int64_t nr = ne1*ne2*ne3;
+
+ // rows per thread
+ const int64_t dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int64_t ir0 = dr*ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
+
+ // block-tiling attempt
+ const int64_t blck_0 = MAX(GGML_VEC_MAD_UNROLL, 32);
+ const int64_t blck_1 = 16;
+
+ // dps == dst per src0, used for group query attention
+ const int64_t dps2 = ne2 / ne02;
+ const int64_t dps3 = ne3 / ne03;
+
+ for (int64_t bir = ir0; bir < ir1; bir += blck_1) {
+ const int64_t bir1 = MIN(bir + blck_1, ir1);
+ for (int64_t bi01 = 0; bi01 < ne01; bi01 += blck_0) {
+ const int64_t bne01 = MIN(bi01 + blck_0, ne01);
+ for (int64_t ir = bir; ir < bir1; ++ir) {
+ // dst indices
+ const int64_t i3 = ir/(ne2*ne1);
+ const int64_t i2 = (ir - i3*ne2*ne1)/ne1;
+ const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ const int64_t i02 = i2 / dps2;
+ const int64_t i03 = i3 / dps3;
+
+ //const int64_t i10 = i1;
+ const int64_t i12 = i2;
+ const int64_t i13 = i3;
+
+ for (int64_t i01 = bi01; i01 < bne01; ++i01) {
+ const int64_t i11 = i01;
+
+ ggml_fp16_t * s0 = (ggml_fp16_t *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03));
+ float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
+ float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
+
+ for (int i = 0; i < ne0; ++i) {
+ d[i] += GGML_CPU_FP16_TO_FP32(s0[i])*(*s1);
+ }
+ }
+ }
+ }
+ }
+}
+
static void ggml_compute_forward_out_prod_q_f32(
const ggml_compute_params * params,
ggml_tensor * dst) {
@@ -4291,9 +4537,8 @@ void ggml_compute_forward_out_prod(
} break;
case GGML_TYPE_F16:
{
- GGML_ABORT("fatal error"); // todo
- // ggml_compute_forward_out_prod_f16_f32(params, dst);
- }
+ ggml_compute_forward_out_prod_f16_f32(params, dst);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_out_prod_f32(params, dst);
@@ -10300,6 +10545,196 @@ void ggml_compute_forward_cross_entropy_loss_back(
}
}
+// ggml_compute_forward_cross_entropy_loss_masked_f32
+
+static void ggml_compute_forward_cross_entropy_loss_masked_f32(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0]; // logits
+ const ggml_tensor * src1 = dst->src[1]; // targets
+ const ggml_tensor * src2 = dst->src[2]; // mask (1 for assistant tokens, 0 for masked)
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT(src2->type == GGML_TYPE_F32);
+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
+ GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));
+ GGML_ASSERT(src2->nb[0] == ggml_type_size(src2->type));
+ GGML_ASSERT(ggml_are_same_shape(src0, src1));
+ GGML_ASSERT(ggml_is_scalar(dst));
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
+
+ const int64_t nc = src0->ne[0];
+ const int64_t nr = ggml_nrows(src0);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ float * sums = (float *) params->wdata;
+ float * st = ((float *) params->wdata) + nth + ith*nc;
+ float sum_thread = 0.0f;
+ int64_t valid_tokens_thread = 0;
+
+ GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc) + sizeof(int64_t) * nth);
+ int64_t * valid_counts = (int64_t *)(((float *) params->wdata) + nth + nth * nc);
+
+ const int64_t dr = (nr + nth - 1)/nth;
+
+ const int64_t ir0 = dr*ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
+
+ for (int64_t i1 = ir0; i1 < ir1; ++i1) {
+ const float * s0 = (const float *)((const char *) src0->data + i1*src0->nb[1]);
+ const float * s1 = (const float *)((const char *) src1->data + i1*src1->nb[1]);
+ const float * mask_row = (const float *)((const char *) src2->data + i1*src2->nb[1]);
+ const float mask_value = mask_row[0];
+
+ if (mask_value <= 0.5f) continue;
+
+ float max = -INFINITY;
+ ggml_vec_max_f32(nc, &max, s0);
+
+ const ggml_float sum_softmax = ggml_vec_log_soft_max_f32(nc, st, s0, max);
+ assert(sum_softmax >= 0.0);
+
+ ggml_vec_add1_f32(nc, st, st, -sum_softmax);
+
+ float sum_st = 0.0f;
+ for (int64_t i = 0; i < nc; i++) {
+ sum_st += st[i] * s1[i];
+ }
+
+ sum_thread += sum_st;
+ valid_tokens_thread++;
+ }
+
+ sums[ith] = sum_thread;
+ valid_counts[ith] = valid_tokens_thread;
+ ggml_barrier(params->threadpool);
+
+ if (ith == 0) {
+ float total_loss = 0.0f;
+ int64_t total_valid = 0;
+
+ for (int i = 0; i < nth; i++) {
+ total_loss += sums[i];
+ total_valid += valid_counts[i];
+ }
+
+ float * dp = (float *) dst->data;
+ if (total_valid > 0) {
+ float final_loss = -total_loss / (float)total_valid;
+ dp[0] = final_loss;
+ } else {
+ dp[0] = 0.0f;
+ }
+ }
+}
+
+// ggml_compute_forward_cross_entropy_loss_masked_back_f32
+
+static void ggml_compute_forward_cross_entropy_loss_masked_back_f32(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+ const ggml_tensor * grad = dst->src[0];
+ const ggml_tensor * src0f = dst->src[1];
+ const ggml_tensor * src1f = dst->src[2];
+ const ggml_tensor * src2f = dst->src[3];
+
+ GGML_ASSERT(ggml_is_contiguous(dst));
+ GGML_ASSERT(ggml_is_contiguous(src0f));
+ GGML_ASSERT(ggml_is_contiguous(src1f));
+ GGML_ASSERT(ggml_is_contiguous(src2f));
+ GGML_ASSERT(ggml_is_contiguous(grad));
+ GGML_ASSERT(ggml_are_same_shape(src0f, src1f) && ggml_are_same_shape(src0f, dst));
+
+ const int64_t ith = params->ith;
+ const int64_t nth = params->nth;
+
+ const int64_t nc = src0f->ne[0];
+ const int64_t nr = ggml_nrows(src0f);
+
+ int64_t total_valid = 0;
+ for (int64_t i1 = 0; i1 < nr; i1++) {
+ const float * mask_row = (const float *)((const char *) src2f->data + i1*src2f->nb[1]);
+ const float mask_value = mask_row[0];
+ if (mask_value > 0.5f) {
+ total_valid++;
+ }
+ }
+
+ const int64_t dr = (nr + nth - 1)/nth;
+
+ const int64_t ir0 = dr*ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
+
+ const float upstream_grad = ((const float *) grad->data)[0];
+
+ float d_scale = 0.0f;
+ if (total_valid > 0) {
+ d_scale = upstream_grad / (float) total_valid;
+ }
+
+ for (int64_t i1 = ir0; i1 < ir1; i1++) {
+ float * ds0 = (float *)((char *) dst->data + i1*dst->nb[1]);
+ const float * s0 = (const float *)((const char *) src0f->data + i1*src0f->nb[1]);
+ const float * s1 = (const float *)((const char *) src1f->data + i1*src1f->nb[1]);
+ const float * mask_row = (const float *)((const char *) src2f->data + i1*src2f->nb[1]);
+ const float mask_value = mask_row[0];
+
+ if (mask_value > 0.5f) {
+ float max = -INFINITY;
+ ggml_vec_max_f32(nc, &max, s0);
+ const ggml_float sum = ggml_vec_soft_max_f32(nc, ds0, s0, max);
+ assert(sum > 0.0);
+ ggml_vec_scale_f32(nc, ds0, 1.0/sum);
+
+ ggml_vec_sub_f32(nc, ds0, ds0, s1);
+ ggml_vec_scale_f32(nc, ds0, d_scale);
+ } else {
+ ggml_vec_set_f32(nc, ds0, 0.0f);
+
+ }
+ }
+}
+
+void ggml_compute_forward_cross_entropy_loss_masked(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_cross_entropy_loss_masked_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
+void ggml_compute_forward_cross_entropy_loss_masked_back(
+ const ggml_compute_params * params,
+ ggml_tensor * dst) {
+
+ const ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_cross_entropy_loss_masked_back_f32(params, dst);
+ } break;
+ default:
+ {
+ GGML_ABORT("fatal error");
+ }
+ }
+}
+
static void ggml_compute_forward_opt_step_adamw_f32(
const ggml_compute_params * params,
ggml_tensor * dst) {
diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h
index 0fdfee79766e..d762ac2c0a6a 100644
--- a/ggml/src/ggml-cpu/ops.h
+++ b/ggml/src/ggml-cpu/ops.h
@@ -38,10 +38,12 @@ void ggml_compute_forward_cumsum(const struct ggml_compute_params * params, stru
void ggml_compute_forward_mean(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_argmax(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_count_equal(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+void ggml_compute_forward_count_equal_masked(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_repeat(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_repeat_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_concat(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+void ggml_compute_forward_geglu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
@@ -108,6 +110,8 @@ void ggml_compute_forward_map_custom3(const struct ggml_compute_params * params,
void ggml_compute_forward_custom(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_cross_entropy_loss(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_cross_entropy_loss_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+void ggml_compute_forward_cross_entropy_loss_masked(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+void ggml_compute_forward_cross_entropy_loss_masked_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_opt_step_adamw(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_mul_mat(const struct ggml_compute_params * params, struct ggml_tensor * dst);
void ggml_compute_forward_opt_step_sgd(const struct ggml_compute_params * params, struct ggml_tensor * dst);
diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c
index 365cb36d2d76..b39278f1b8e5 100644
--- a/ggml/src/ggml-cpu/quants.c
+++ b/ggml/src/ggml-cpu/quants.c
@@ -332,6 +332,35 @@ void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c
*s = sumf;
}
+void ggml_vec_dot_q8_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ const int qk = QK8_1;
+ const int nb = n / qk;
+
+ assert(n % qk == 0);
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ const block_q8_1 * GGML_RESTRICT x = vx;
+ const block_q8_1 * GGML_RESTRICT y = vy;
+
+ float sumf = 0;
+
+ for (int ib = 0; ib < nb; ++ib) {
+ int sumi = 0;
+
+ for (int j = 0; j < qk; ++j) {
+ sumi += x[ib].qs[j]*y[ib].qs[j];
+ }
+
+ sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));
+ }
+
+ *s = sumf;
+}
+
void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
assert(nrc == 1);
UNUSED(nrc);
@@ -416,6 +445,90 @@ void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs,
*s = sumf;
}
+void ggml_vec_dot_tq2_0_q8_0_generic(
+ int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx,
+ const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ GGML_ASSERT(n % QK_K == 0);
+ const int nb = n / QK_K;
+ const int q8_blocks = QK_K / QK8_0;
+
+ const block_tq2_0 * GGML_RESTRICT x = vx;
+ const block_q8_0 * GGML_RESTRICT y = vy;
+
+ float sumf = 0.0f;
+
+ for (int ib = 0; ib < nb; ++ib) {
+ const float dx = GGML_CPU_FP16_TO_FP32(x[ib].d);
+ const block_q8_0 * y_blocks = y + ib * q8_blocks;
+
+ size_t block_index = 0;
+ for (size_t j = 0; j < sizeof(x[ib].qs); j += 32) {
+ for (int l = 0; l < 4; ++l) {
+ const block_q8_0 * yb = &y_blocks[block_index++];
+ const float dy = GGML_CPU_FP16_TO_FP32(yb->d);
+
+ int32_t sumi = 0;
+ for (int k = 0; k < QK8_0; ++k) {
+ const int8_t xv = ((x[ib].qs[j + k] >> (l*2)) & 3) - 1;
+ sumi += xv * yb->qs[k];
+ }
+
+ sumf += (float) sumi * dx * dy;
+ }
+ }
+ }
+
+ *s = sumf;
+}
+
+void ggml_vec_dot_tq2_0_q8_1_generic(
+ int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx,
+ const void * GGML_RESTRICT vy, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+
+ GGML_ASSERT(n % QK_K == 0);
+ const int nb = n / QK_K;
+ const int q8_blocks = QK_K / QK8_1;
+
+ const block_tq2_0 * GGML_RESTRICT x = vx;
+ const block_q8_1 * GGML_RESTRICT y = vy;
+
+ float sumf = 0.0f;
+
+ for (int ib = 0; ib < nb; ++ib) {
+ const float dx = GGML_CPU_FP16_TO_FP32(x[ib].d);
+ const block_q8_1 * y_blocks = y + ib * q8_blocks;
+
+ size_t block_index = 0;
+ for (size_t j = 0; j < sizeof(x[ib].qs); j += 32) {
+ for (int l = 0; l < 4; ++l) {
+ const block_q8_1 * yb = &y_blocks[block_index++];
+ const float dy = GGML_CPU_FP16_TO_FP32(yb->d);
+
+ int32_t sumi = 0;
+ for (int k = 0; k < QK8_1; ++k) {
+ const int8_t xv = ((x[ib].qs[j + k] >> (l*2)) & 3) - 1;
+ sumi += xv * yb->qs[k];
+ }
+
+ sumf += (float) sumi * dx * dy;
+ }
+ }
+ }
+
+ *s = sumf;
+}
+
void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
assert(nrc == 1);
UNUSED(nrc);
diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h
index d83eb1b144d4..d859bf0234d7 100644
--- a/ggml/src/ggml-cpu/quants.h
+++ b/ggml/src/ggml-cpu/quants.h
@@ -40,6 +40,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q8_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -51,6 +52,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq2_0_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
@@ -71,11 +74,14 @@ void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, c
void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_q8_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
+void ggml_vec_dot_tq2_0_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h
index bd80805fdc55..a2c6be8b3200 100644
--- a/ggml/src/ggml-cpu/vec.h
+++ b/ggml/src/ggml-cpu/vec.h
@@ -1310,6 +1310,32 @@ inline static void ggml_vec_silu_backward_f16(const int n, ggml_fp16_t * dx, con
}
}
+inline static float ggml_gelu_backward_f32(float x, float dy) {
+ const float tanh_arg = SQRT_2_OVER_PI * x * (1.0f + GELU_COEF_A * x * x);
+ const float tanh_val = tanhf(tanh_arg);
+ const float sech2_val = 1.0f - tanh_val * tanh_val;
+ const float dtanh_dx = SQRT_2_OVER_PI * (1.0f + 3.0f * GELU_COEF_A * x * x) * sech2_val;
+ return dy * 0.5f * (1.0f + tanh_val + x * dtanh_dx);
+}
+
+inline static void ggml_vec_gelu_backward_f32(const int n, float * dx, const float * x, const float * dy) {
+ for (int i = 0; i < n; ++i) {
+ dx[i] = ggml_gelu_backward_f32(x[i], dy[i]);
+ }
+}
+
+inline static void ggml_vec_gelu_backward_f16(const int n, ggml_fp16_t * dx, const ggml_fp16_t * x, const ggml_fp16_t * dy) {
+ for (int i = 0; i < n; ++i) {
+ float xi = GGML_CPU_FP16_TO_FP32(x[i]);
+ float tanh_arg = SQRT_2_OVER_PI * xi * (1.0f + GELU_COEF_A * xi * xi);
+ float tanh_val = tanhf(tanh_arg);
+ float sech2_val = 1.0f - tanh_val * tanh_val;
+ float dtanh_dx = SQRT_2_OVER_PI * (1.0f + 3.0f * GELU_COEF_A * xi * xi) * sech2_val;
+
+ dx[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(dy[i]) * 0.5f * (1.0f + tanh_val + xi * dtanh_dx));
+ }
+}
+
inline static void ggml_vec_reglu_f32 (const int n, float * y, const float * x, const float * g) {
for (int i = 0; i < n; ++i) {
y[i] = (x[i] > 0.f) ? x[i] * g[i] : 0.f;
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 88352a92aa28..988050c9fb91 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -4385,7 +4385,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
}
} break;
case GGML_OP_OUT_PROD:
- return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32;
+ return op->type == GGML_TYPE_F32;
case GGML_OP_GET_ROWS:
{
switch (op->src[0]->type) {
diff --git a/ggml/src/ggml-cuda/out-prod.cu b/ggml/src/ggml-cuda/out-prod.cu
index c9b2b699c6a5..f1a92be302dc 100644
--- a/ggml/src/ggml-cuda/out-prod.cu
+++ b/ggml/src/ggml-cuda/out-prod.cu
@@ -1,4 +1,5 @@
#include "out-prod.cuh"
+#include "convert.cuh"
#include
@@ -8,10 +9,56 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
GGML_TENSOR_BINARY_OP_LOCALS
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
- GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ const bool src0_is_quantized = (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16);
+ const bool src1_is_quantized = (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16);
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
+ cudaStream_t stream = ctx.stream();
+ ggml_cuda_pool & pool = ctx.pool();
+
+ // temp buffers
+ float * src0_f32 = nullptr;
+ float * src1_f32 = nullptr;
+ bool allocated_src0 = false;
+ bool allocated_src1 = false;
+ ggml_cuda_pool_alloc src0_alloc(pool);
+ ggml_cuda_pool_alloc src1_alloc(pool);
+
+ if (src0_is_quantized) {
+ const size_t src0_size = ggml_nelements(src0);
+ src0_alloc.alloc(src0_size);
+ src0_f32 = src0_alloc.ptr;
+ allocated_src0 = true;
+
+ // Dequantize
+ auto dequantize_fn = ggml_get_to_fp32_cuda(src0->type);
+ if (dequantize_fn) {
+ dequantize_fn(src0->data, src0_f32, ggml_nelements(src0), stream);
+ } else {
+ GGML_ABORT("Unsupported quant type for src0");
+ }
+ } else {
+ src0_f32 = (float *) src0->data;
+ }
+
+ if (src1_is_quantized) {
+ const size_t src1_size = ggml_nelements(src1);
+ src1_alloc.alloc(src1_size);
+ src1_f32 = src1_alloc.ptr;
+ allocated_src1 = true;
+
+ auto dequantize_fn = ggml_get_to_fp32_cuda(src1->type);
+ if (dequantize_fn) {
+ dequantize_fn(src1->data, src1_f32, ggml_nelements(src1), stream);
+ } else {
+ GGML_ABORT("Unsupported quant type for src1");
+ }
+ } else {
+ src1_f32 = (float *) src1->data;
+ }
+
+
GGML_ASSERT(ne01 == ne11);
GGML_ASSERT(ne0 == ne00);
GGML_ASSERT(ne1 == ne10);
@@ -22,11 +69,11 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
GGML_ASSERT(ne2 == src1->ne[2]);
GGML_ASSERT(ne3 == src1->ne[3]);
- const float * src0_d = (const float *) src0->data;
- const float * src1_d = (const float *) src1->data;
+ // Use dequantized data
+ const float * src0_d = src0_f32;
+ const float * src1_d = src1_f32;
float * dst_d = (float *) dst->data;
- cudaStream_t stream = ctx.stream();
cublasHandle_t handle = ctx.cublas_handle();
const float alpha = 1.0f;
@@ -34,19 +81,25 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
CUBLAS_CHECK(cublasSetStream(handle, stream));
- const int64_t lda = nb01 / sizeof(float);
+ const int64_t lda = allocated_src0 ? ne00 : (nb01 / sizeof(float));
const int64_t ldc = nb1 / sizeof(float);
const bool src1_T = ggml_is_transposed(src1);
const cublasOperation_t src1_cublas_op = src1_T ? CUBLAS_OP_N : CUBLAS_OP_T;
- const int64_t ldb = (src1_T ? nb10 : nb11) / sizeof(float);
- GGML_ASSERT( (src1_T ? nb11 : nb10) == sizeof(float));
+ const int64_t ldb = allocated_src1 ?
+ (src1_T ? ne10 : ne11) :
+ ((src1_T ? nb10 : nb11) / sizeof(float));
+
+ // Only assert for non dequantized src1
+ if (!allocated_src1) {
+ GGML_ASSERT((src1_T ? nb11 : nb10) == sizeof(float));
+ }
// data strides in dimensions 2/3
- const size_t s02 = nb02 / sizeof(float);
- const size_t s03 = nb03 / sizeof(float);
- const size_t s12 = nb12 / sizeof(float);
- const size_t s13 = nb13 / sizeof(float);
+ const size_t s02 = allocated_src0 ? (ne00 * ne01) : nb02 / sizeof(float);
+ const size_t s03 = allocated_src0 ? (ne00 * ne01 * ne02): nb03 / sizeof(float);
+ const size_t s12 = allocated_src1 ? (ne10 * ne11) : nb12 / sizeof(float);
+ const size_t s13 = allocated_src1 ? (ne10 * ne11 * ne12) : nb13 / sizeof(float);
const size_t s2 = nb2 / sizeof(float);
const size_t s3 = nb3 / sizeof(float);
@@ -65,4 +118,5 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
&beta, dst_d + i3 *s3 + i2 *s2, ldc));
}
}
+
}
diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m
index e66646284dbc..8587317c2e0f 100644
--- a/ggml/src/ggml-metal/ggml-metal-context.m
+++ b/ggml/src/ggml-metal/ggml-metal-context.m
@@ -8,6 +8,7 @@
#import "ggml-metal-ops.h"
#import
+#import
#import
@@ -127,6 +128,12 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) {
res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil;
res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil;
+#if TARGET_OS_IPHONE
+ if (getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil) {
+ res->use_concurrency = false;
+ }
+#endif
+
{
const char * val = getenv("GGML_METAL_GRAPH_DEBUG");
res->debug_graph = val ? atoi(val) : 0;
@@ -536,6 +543,7 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
}
ctx->encode_async = Block_copy(^(size_t iter) {
+ @autoreleasepool {
const int cb_idx = iter;
const int n_cb_l = ctx->n_cb;
@@ -580,6 +588,7 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) {
if (cb_idx < 2 || ctx->abort_callback == NULL) {
[cmd_buf commit];
}
+ } // @autoreleasepool
});
}
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index 329500a03e0d..112afd32c29a 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -597,6 +597,12 @@ ggml_metal_pipeline_t ggml_metal_library_get_pipeline_mul_mv(ggml_metal_library_
nr0 = N_R0_Q8_0;
smem = 32*sizeof(float)*N_R0_Q8_0;
} break;
+ case GGML_TYPE_TQ2_0:
+ {
+ nsg = N_SG_TQ2_0;
+ nr0 = N_R0_TQ2_0;
+ smem = 32*sizeof(float)*N_R0_TQ2_0;
+ } break;
case GGML_TYPE_MXFP4:
{
nsg = N_SG_MXFP4;
@@ -817,6 +823,12 @@ ggml_metal_pipeline_t ggml_metal_library_get_pipeline_mul_mv_id(ggml_metal_libra
nr0 = N_R0_Q8_0;
smem = 32*sizeof(float)*N_R0_Q8_0;
} break;
+ case GGML_TYPE_TQ2_0:
+ {
+ nsg = N_SG_TQ2_0;
+ nr0 = N_R0_TQ2_0;
+ smem = 32*sizeof(float)*N_R0_TQ2_0;
+ } break;
case GGML_TYPE_MXFP4:
{
nsg = N_SG_MXFP4;
@@ -1681,6 +1693,111 @@ ggml_metal_pipeline_t ggml_metal_library_get_pipeline_timestep_embedding(ggml_me
return res;
}
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_out_prod(ggml_metal_library_t lib, const ggml_tensor * op) {
+ assert(op->op == GGML_OP_OUT_PROD);
+ assert(op->type == GGML_TYPE_F32);
+
+ char base[256];
+ char name[256];
+
+ const ggml_type src0t = op->src[0]->type;
+ const ggml_type src1t = op->src[1]->type;
+
+ if (src0t == GGML_TYPE_Q8_0) {
+ snprintf(base, 256, "kernel_out_prod_q8_0_%s", ggml_type_name(src1t));
+ } else if (src0t == GGML_TYPE_Q4_0) {
+ snprintf(base, 256, "kernel_out_prod_q4_0_%s", ggml_type_name(src1t));
+ } else if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32) {
+ snprintf(base, 256, "kernel_out_prod_f16_f32");
+ } else if (src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32) {
+ snprintf(base, 256, "kernel_out_prod_f32");
+ } else if (src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F16) {
+ snprintf(base, 256, "kernel_out_prod_f32_f16");
+ } else if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F16) {
+ snprintf(base, 256, "kernel_out_prod_f16");
+ } else {
+ GGML_ABORT("fatal error");
+ }
+
+ snprintf(name, 256, "%s", base);
+
+ ggml_metal_pipeline_t res = ggml_metal_library_get_pipeline(lib, name);
+ if (res) {
+ return res;
+ }
+
+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+
+ return res;
+}
+
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_silu_back(ggml_metal_library_t lib, const ggml_tensor * op) {
+ assert(op->op == GGML_OP_SILU_BACK);
+
+ char base[256];
+ char name[256];
+
+ const int64_t n = ggml_nelements(op);
+ const char * suffix = (n % 4 == 0) ? "_4" : "";
+
+ snprintf(base, 256, "kernel_silu_back%s", suffix);
+ snprintf(name, 256, "%s", base);
+
+ ggml_metal_pipeline_t res = ggml_metal_library_get_pipeline(lib, name);
+ if (res) {
+ return res;
+ }
+
+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+
+ return res;
+}
+
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_soft_max_back(ggml_metal_library_t lib, const ggml_tensor * op) {
+ assert(op->op == GGML_OP_SOFT_MAX_BACK);
+
+ char base[256];
+ char name[256];
+
+ const char * suffix = (op->src[0]->ne[0] % 4 == 0) ? "_4" : "";
+
+ snprintf(base, 256, "kernel_soft_max_back%s", suffix);
+ snprintf(name, 256, "%s", base);
+
+ ggml_metal_pipeline_t res = ggml_metal_library_get_pipeline(lib, name);
+ if (res) {
+ return res;
+ }
+
+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+
+ ggml_metal_pipeline_set_smem(res, 32*sizeof(float));
+
+ return res;
+}
+
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_rms_norm_back(ggml_metal_library_t lib, const ggml_tensor * op) {
+ assert(op->op == GGML_OP_RMS_NORM_BACK);
+ GGML_UNUSED(op);
+
+ char base[256];
+ char name[256];
+
+ snprintf(base, 256, "kernel_rms_norm_back");
+ snprintf(name, 256, "%s", base);
+
+ ggml_metal_pipeline_t res = ggml_metal_library_get_pipeline(lib, name);
+ if (res) {
+ return res;
+ }
+
+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+
+ ggml_metal_pipeline_set_smem(res, 2*32*sizeof(float));
+
+ return res;
+}
+
ggml_metal_pipeline_t ggml_metal_library_get_pipeline_opt_step_adamw(ggml_metal_library_t lib, const ggml_tensor * op) {
assert(op->op == GGML_OP_OPT_STEP_ADAMW);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index 3976e622b9b9..d1dde6fd4325 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -144,8 +144,12 @@ ggml_metal_pipeline_t ggml_metal_library_get_pipeline_pad (ggml_me
ggml_metal_pipeline_t ggml_metal_library_get_pipeline_pad_reflect_1d (ggml_metal_library_t lib, const struct ggml_tensor * op);
ggml_metal_pipeline_t ggml_metal_library_get_pipeline_arange (ggml_metal_library_t lib, const struct ggml_tensor * op);
ggml_metal_pipeline_t ggml_metal_library_get_pipeline_timestep_embedding(ggml_metal_library_t lib, const struct ggml_tensor * op);
-ggml_metal_pipeline_t ggml_metal_library_get_pipeline_opt_step_adamw (ggml_metal_library_t lib, const struct ggml_tensor * op);
-ggml_metal_pipeline_t ggml_metal_library_get_pipeline_opt_step_sgd (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_out_prod (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_silu_back (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_soft_max_back (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_rms_norm_back (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_opt_step_adamw (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ggml_metal_pipeline_t ggml_metal_library_get_pipeline_opt_step_sgd (ggml_metal_library_t lib, const struct ggml_tensor * op);
ggml_metal_pipeline_t ggml_metal_library_get_pipeline_flash_attn_ext_pad(
ggml_metal_library_t lib,
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 62bc4ba45fcc..15a453ea3571 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -849,6 +849,31 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_OP_DIV:
case GGML_OP_ADD_ID:
return op->src[0]->type == GGML_TYPE_F32;
+ case GGML_OP_OUT_PROD:
+ if (op->type != GGML_TYPE_F32) {
+ return false;
+ }
+ {
+ const enum ggml_type src0_type = op->src[0]->type;
+ const enum ggml_type src1_type = op->src[1]->type;
+
+ if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F32 && (src1_type == GGML_TYPE_F32 || src1_type == GGML_TYPE_F16)) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_Q8_0 && (src1_type == GGML_TYPE_F32 || src1_type == GGML_TYPE_F16)) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_Q4_0 && (src1_type == GGML_TYPE_F32 || src1_type == GGML_TYPE_F16)) {
+ return true;
+ }
+ }
+ return false;
case GGML_OP_ACC:
case GGML_OP_REPEAT:
case GGML_OP_SCALE:
@@ -861,6 +886,16 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
op->type == GGML_TYPE_F32;
case GGML_OP_CLAMP:
return op->src[0]->type == GGML_TYPE_F32;
+ case GGML_OP_SILU_BACK:
+ return op->type == GGML_TYPE_F32 &&
+ op->src[0] != NULL && op->src[1] != NULL &&
+ op->src[0]->type == GGML_TYPE_F32 &&
+ op->src[1]->type == GGML_TYPE_F32 &&
+ ggml_is_contiguous_1(op->src[0]) &&
+ ggml_is_contiguous_1(op->src[1]) &&
+ ggml_is_contiguous_1(op) &&
+ ggml_are_same_shape(op, op->src[0]) &&
+ ggml_are_same_shape(op, op->src[1]);
case GGML_OP_SQR:
case GGML_OP_SQRT:
case GGML_OP_SIN:
@@ -875,6 +910,29 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_OP_SOFT_MAX:
case GGML_OP_GROUP_NORM:
return has_simdgroup_reduction && ggml_is_contiguous_rows(op->src[0]);
+ case GGML_OP_SOFT_MAX_BACK:
+ {
+ if (!has_simdgroup_reduction ||
+ op->type != GGML_TYPE_F32 ||
+ op->src[0] == NULL || op->src[1] == NULL ||
+ op->src[0]->type != GGML_TYPE_F32 ||
+ op->src[1]->type != GGML_TYPE_F32 ||
+ !ggml_is_contiguous_1(op->src[0]) ||
+ !ggml_is_contiguous_1(op->src[1]) ||
+ !ggml_is_contiguous_1(op) ||
+ !ggml_are_same_shape(op, op->src[0]) ||
+ !ggml_are_same_shape(op, op->src[1])) {
+ return false;
+ }
+
+ float max_bias = 0.0f;
+ memcpy(&max_bias, ((const float *) op->op_params) + 1, sizeof(float));
+ if (max_bias != 0.0f) {
+ return false;
+ }
+
+ return true;
+ }
case GGML_OP_L2_NORM:
return has_simdgroup_reduction && (op->ne[0] % 4 == 0 && ggml_is_contiguous_1(op->src[0]));
case GGML_OP_ARGMAX:
@@ -882,6 +940,18 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
case GGML_OP_NORM:
case GGML_OP_RMS_NORM:
return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0]));
+ case GGML_OP_RMS_NORM_BACK:
+ return has_simdgroup_reduction &&
+ op->type == GGML_TYPE_F32 &&
+ op->src[0] != NULL && op->src[1] != NULL &&
+ op->src[0]->type == GGML_TYPE_F32 &&
+ op->src[1]->type == GGML_TYPE_F32 &&
+ op->ne[0] % 4 == 0 &&
+ ggml_is_contiguous_1(op->src[0]) &&
+ ggml_is_contiguous_1(op->src[1]) &&
+ ggml_is_contiguous_1(op) &&
+ ggml_are_same_shape(op, op->src[0]) &&
+ ggml_are_same_shape(op, op->src[1]);
case GGML_OP_ROPE:
return true;
case GGML_OP_IM2COL:
@@ -940,7 +1010,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
return true;
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
- return has_simdgroup_reduction;
+ return op->src[0]->type != GGML_TYPE_TQ1_0 && has_simdgroup_reduction;
case GGML_OP_CPY:
case GGML_OP_DUP:
case GGML_OP_CONT:
@@ -997,7 +1067,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
};
}
case GGML_OP_GET_ROWS:
- return true;
+ return op->src[0]->type != GGML_TYPE_TQ1_0;
case GGML_OP_SET_ROWS:
{
if (op->src[0]->type != GGML_TYPE_F32) {
diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h
index 342dc4f8c378..e63e95d856fe 100644
--- a/ggml/src/ggml-metal/ggml-metal-impl.h
+++ b/ggml/src/ggml-metal/ggml-metal-impl.h
@@ -23,6 +23,9 @@
#define N_R0_Q8_0 2
#define N_SG_Q8_0 4
+#define N_R0_Q8_1 2
+#define N_SG_Q8_1 4
+
#define N_R0_MXFP4 2
#define N_SG_MXFP4 2
@@ -41,6 +44,9 @@
#define N_R0_Q6_K 2
#define N_SG_Q6_K 2
+#define N_R0_TQ2_0 4
+#define N_SG_TQ2_0 2
+
#define N_R0_IQ1_S 4
#define N_SG_IQ1_S 2
@@ -207,6 +213,33 @@ typedef struct {
uint64_t nb3;
} ggml_metal_kargs_cpy;
+typedef struct {
+ int32_t ne00;
+ int32_t ne01;
+ int32_t ne02;
+ int32_t ne03;
+ uint64_t nb00;
+ uint64_t nb01;
+ uint64_t nb02;
+ uint64_t nb03;
+ int32_t ne10;
+ int32_t ne11;
+ int32_t ne12;
+ int32_t ne13;
+ uint64_t nb10;
+ uint64_t nb11;
+ uint64_t nb12;
+ uint64_t nb13;
+ int32_t ne0;
+ int32_t ne1;
+ int32_t ne2;
+ int32_t ne3;
+ uint64_t nb0;
+ uint64_t nb1;
+ uint64_t nb2;
+ uint64_t nb3;
+} ggml_metal_kargs_out_prod;
+
typedef struct {
int64_t ne10;
int64_t ne11;
@@ -253,6 +286,7 @@ typedef struct {
int32_t sect_2;
int32_t sect_3;
bool src2;
+ float sin_sign;
} ggml_metal_kargs_rope;
typedef struct {
@@ -488,6 +522,21 @@ typedef struct {
uint64_t nbf3[3];
} ggml_metal_kargs_norm;
+typedef struct {
+ int32_t ne00;
+ int32_t ne00_4;
+ uint64_t nb01;
+ uint64_t nb02;
+ uint64_t nb03;
+ uint64_t nb11;
+ uint64_t nb12;
+ uint64_t nb13;
+ uint64_t nb1;
+ uint64_t nb2;
+ uint64_t nb3;
+ float eps;
+} ggml_metal_kargs_rms_norm_back;
+
typedef struct {
int32_t ne00;
int32_t ne00_4;
@@ -674,6 +723,21 @@ typedef struct {
int32_t n_head_log2;
} ggml_metal_kargs_soft_max;
+typedef struct {
+ int32_t ne00;
+ int32_t ne00_4;
+ uint64_t nb01;
+ uint64_t nb02;
+ uint64_t nb03;
+ uint64_t nb11;
+ uint64_t nb12;
+ uint64_t nb13;
+ uint64_t nb1;
+ uint64_t nb2;
+ uint64_t nb3;
+ float scale;
+} ggml_metal_kargs_soft_max_back;
+
typedef struct {
int64_t ne00;
int64_t ne01;
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 9871e976f23d..479948f809ab 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -278,6 +278,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
{
n_fuse = ggml_metal_op_repeat(ctx, idx);
} break;
+ case GGML_OP_OUT_PROD:
+ {
+ n_fuse = ggml_metal_op_out_prod(ctx, idx);
+ } break;
case GGML_OP_ACC:
{
n_fuse = ggml_metal_op_acc(ctx, idx);
@@ -299,6 +303,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
{
n_fuse = ggml_metal_op_unary(ctx, idx);
} break;
+ case GGML_OP_SILU_BACK:
+ {
+ n_fuse = ggml_metal_op_silu_back(ctx, idx);
+ } break;
case GGML_OP_GLU:
{
n_fuse = ggml_metal_op_glu(ctx, idx);
@@ -320,6 +328,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
{
n_fuse = ggml_metal_op_soft_max(ctx, idx);
} break;
+ case GGML_OP_SOFT_MAX_BACK:
+ {
+ n_fuse = ggml_metal_op_soft_max_back(ctx, idx);
+ } break;
case GGML_OP_SSM_CONV:
{
n_fuse = ggml_metal_op_ssm_conv(ctx, idx);
@@ -362,6 +374,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
{
n_fuse = ggml_metal_op_norm(ctx, idx);
} break;
+ case GGML_OP_RMS_NORM_BACK:
+ {
+ n_fuse = ggml_metal_op_rms_norm_back(ctx, idx);
+ } break;
case GGML_OP_ROPE:
{
n_fuse = ggml_metal_op_rope(ctx, idx);
@@ -3125,6 +3141,7 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) {
/* sect_2 =*/ sect_2,
/* sect_3 =*/ sect_3,
/* src2 =*/ op->src[2] != nullptr,
+ /* sin_sign =*/ 1.0f,
};
ggml_metal_pipeline_t pipeline = ggml_metal_library_get_pipeline_rope(lib, op);
@@ -3899,6 +3916,218 @@ int ggml_metal_op_leaky_relu(ggml_metal_op_t ctx, int idx) {
return 1;
}
+int ggml_metal_op_out_prod(ggml_metal_op_t ctx, int idx) {
+ ggml_tensor * op = ctx->node(idx);
+
+ ggml_metal_library_t lib = ctx->lib;
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
+ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);
+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);
+
+ ggml_metal_kargs_out_prod args = {
+ /*.ne00 =*/ ne00,
+ /*.ne01 =*/ ne01,
+ /*.ne02 =*/ ne02,
+ /*.ne03 =*/ ne03,
+ /*.nb00 =*/ nb00,
+ /*.nb01 =*/ nb01,
+ /*.nb02 =*/ nb02,
+ /*.nb03 =*/ nb03,
+ /*.ne10 =*/ ne10,
+ /*.ne11 =*/ ne11,
+ /*.ne12 =*/ ne12,
+ /*.ne13 =*/ ne13,
+ /*.nb10 =*/ nb10,
+ /*.nb11 =*/ nb11,
+ /*.nb12 =*/ nb12,
+ /*.nb13 =*/ nb13,
+ /*.ne0 =*/ ne0,
+ /*.ne1 =*/ ne1,
+ /*.ne2 =*/ ne2,
+ /*.ne3 =*/ ne3,
+ /*.nb0 =*/ nb0,
+ /*.nb1 =*/ nb1,
+ /*.nb2 =*/ nb2,
+ /*.nb3 =*/ nb3,
+ };
+
+ ggml_metal_pipeline_t pipeline = ggml_metal_library_get_pipeline_out_prod(lib, op);
+
+ const int threads = ne0 < 1 ? 1 : (int) ne0;
+ const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), threads);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1);
+
+ return 1;
+}
+
+int ggml_metal_op_silu_back(ggml_metal_op_t ctx, int idx) {
+ ggml_tensor * op = ctx->node(idx);
+
+ ggml_metal_library_t lib = ctx->lib;
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ ggml_metal_pipeline_t pipeline = ggml_metal_library_get_pipeline_silu_back(lib, op);
+
+ int64_t n = ggml_nelements(op);
+
+ if (n % 4 == 0) {
+ n /= 4;
+ }
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, 1, 1, 1);
+
+ return 1;
+}
+
+int ggml_metal_op_soft_max_back(ggml_metal_op_t ctx, int idx) {
+ ggml_tensor * op = ctx->node(idx);
+
+ ggml_metal_library_t lib = ctx->lib;
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
+ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);
+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);
+
+ float scale = 1.0f;
+ float max_bias = 0.0f;
+
+ memcpy(&scale, ((const int32_t *) op->op_params) + 0, sizeof(scale));
+ memcpy(&max_bias, ((const int32_t *) op->op_params) + 1, sizeof(max_bias));
+
+ GGML_ASSERT(max_bias == 0.0f);
+
+ const bool use_vec4 = (ne00 % 4) == 0;
+
+ ggml_metal_kargs_soft_max_back args = {
+ /*.ne00 =*/ ne00,
+ /*.ne00_4 =*/ ne00/4,
+ /*.nb01 =*/ nb01,
+ /*.nb02 =*/ nb02,
+ /*.nb03 =*/ nb03,
+ /*.nb11 =*/ nb11,
+ /*.nb12 =*/ nb12,
+ /*.nb13 =*/ nb13,
+ /*.nb1 =*/ nb1,
+ /*.nb2 =*/ nb2,
+ /*.nb3 =*/ nb3,
+ /*.scale =*/ scale,
+ };
+
+ ggml_metal_pipeline_t pipeline = ggml_metal_library_get_pipeline_soft_max_back(lib, op);
+
+ int nth = 32; // SIMD width
+
+ if (use_vec4) {
+ const int ne00_4 = ne00/4;
+ while (nth < ne00_4 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
+ nth *= 2;
+ }
+ nth = std::min(nth, ne00_4);
+ } else {
+ while (nth < ne00 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
+ nth *= 2;
+ }
+ nth = std::min(nth, ne00);
+ }
+
+ nth = std::max(1, nth);
+ nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
+
+ const size_t smem = ggml_metal_pipeline_get_smem(pipeline);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
+
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);
+
+ return 1;
+}
+
+int ggml_metal_op_rms_norm_back(ggml_metal_op_t ctx, int idx) {
+ ggml_tensor * op = ctx->node(idx);
+
+ ggml_metal_library_t lib = ctx->lib;
+ ggml_metal_encoder_t enc = ctx->enc;
+
+ GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
+ GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);
+ GGML_TENSOR_LOCALS( int32_t, ne, op, ne);
+ GGML_TENSOR_LOCALS(uint64_t, nb, op, nb);
+
+ GGML_ASSERT(ne00 % 4 == 0);
+
+ float eps;
+ memcpy(&eps, op->op_params, sizeof(float));
+
+ ggml_metal_kargs_rms_norm_back args = {
+ /*.ne00 =*/ ne00,
+ /*.ne00_4 =*/ ne00/4,
+ /*.nb01 =*/ nb01,
+ /*.nb02 =*/ nb02,
+ /*.nb03 =*/ nb03,
+ /*.nb11 =*/ nb11,
+ /*.nb12 =*/ nb12,
+ /*.nb13 =*/ nb13,
+ /*.nb1 =*/ nb1,
+ /*.nb2 =*/ nb2,
+ /*.nb3 =*/ nb3,
+ /*.eps =*/ eps,
+ };
+
+ ggml_metal_pipeline_t pipeline = ggml_metal_library_get_pipeline_rms_norm_back(lib, op);
+
+ int nth = 32; // SIMD width
+
+ while (nth < ne00/4 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
+ nth *= 2;
+ }
+
+ nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
+ nth = std::min(nth, ne00/4);
+
+ const size_t smem = ggml_metal_pipeline_get_smem(pipeline);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
+
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1);
+
+ return 1;
+}
+
int ggml_metal_op_opt_step_adamw(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h
index b5546146e13d..423b5bf654c5 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.h
+++ b/ggml/src/ggml-metal/ggml-metal-ops.h
@@ -83,8 +83,12 @@ int ggml_metal_op_argmax (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_argsort (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_top_k (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_leaky_relu (ggml_metal_op_t ctx, int idx);
-int ggml_metal_op_opt_step_adamw (ggml_metal_op_t ctx, int idx);
-int ggml_metal_op_opt_step_sgd (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_out_prod (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_silu_back (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_soft_max_back (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_rms_norm_back (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_opt_step_adamw (ggml_metal_op_t ctx, int idx);
+int ggml_metal_op_opt_step_sgd (ggml_metal_op_t ctx, int idx);
#ifdef __cplusplus
}
diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp
index 70bf6f3d981f..3dfc115d95bb 100644
--- a/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/ggml/src/ggml-metal/ggml-metal.cpp
@@ -464,6 +464,20 @@ static ggml_guid_t ggml_backend_metal_guid(void) {
return &guid;
}
+static int ggml_backend_metal_default_n_cb(void) {
+ const char * env_n_cb = getenv("GGML_METAL_N_CB");
+ if (env_n_cb != nullptr) {
+ const int n_cb = atoi(env_n_cb);
+ return n_cb > 0 ? n_cb : 1;
+ }
+
+#if defined(__APPLE__) && defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__)
+ return 2;
+#else
+ return 1;
+#endif
+}
+
ggml_backend_t ggml_backend_metal_init(void) {
ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_metal_reg(), 0);
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
@@ -483,7 +497,7 @@ ggml_backend_t ggml_backend_metal_init(void) {
/* .context = */ ctx,
};
- ggml_backend_metal_set_n_cb(backend, 1);
+ ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb());
return backend;
}
@@ -575,7 +589,7 @@ static ggml_backend_t ggml_backend_metal_device_init(ggml_backend_dev_t dev, con
/* .context = */ ctx,
};
- ggml_backend_metal_set_n_cb(backend, 1);
+ ggml_backend_metal_set_n_cb(backend, ggml_backend_metal_default_n_cb());
return backend;
diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal
index 3ca8d9b322b0..cf94c12f5fe8 100644
--- a/ggml/src/ggml-metal/ggml-metal.metal
+++ b/ggml/src/ggml-metal/ggml-metal.metal
@@ -129,6 +129,126 @@ void dequantize_q4_0(device const block_q4_0 * xb, short il, thread type4x4 & re
reg = (type4x4) reg_f;
}
+template
+void dequantize_tq2_0(device const block_tq2_0 * xb, short il, thread type4x4 & reg) {
+ const half d = xb->d;
+ device const uint8_t * q = (device const uint8_t *) xb->qs + 32*(il/8) + 16*(il & 1);
+
+ il = (il/2)%4;
+
+ const half coef_lut[4] = { 1.h, 1/4.h, 1/16.h, 1/64.h };
+ const uchar mask_lut[4] = { 3, 12, 48, 192 };
+ half coef = coef_lut[il];
+ uchar mask = mask_lut[il];
+ const half dl = d * coef;
+ for (int i = 0; i < 16; ++i) {
+ reg[i/4][i%4] = dl * (q[i] & mask) - d;
+ }
+}
+
+template
+void kernel_mul_mv_tq2_0_f32_impl(
+ args_t args,
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ threadgroup char * shmem,
+ uint3 tgpig,
+ ushort tiisg,
+ ushort sgitg) {
+
+ const int nb = args.ne00/QK_K;
+ const int r0 = tgpig.x;
+ const int r1 = tgpig.y;
+ const int im = tgpig.z;
+
+ const int first_row = (r0 * N_SG_TQ2_0 + sgitg) * N_R0_TQ2_0;
+
+ const uint i12 = im%args.ne12;
+ const uint i13 = im/args.ne12;
+
+ const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03;
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
+
+ device const block_tq2_0 * x = (device const block_tq2_0 *) (src0 + offset0);
+ device const float * y = (device const float *) (src1 + offset1);
+
+ float yl[32];
+ float sumf[N_R0_TQ2_0] = {0.f};
+ float all_sum;
+
+ const int step = sizeof(block_tq2_0) * nb / 2;
+
+ const int ix = tiisg/8; // 0...3
+ const int it = tiisg%8; // 0...7
+ const int iq = it/4; // 0 or 1
+ const int ir = it%4; // 0...3
+
+ device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;
+
+ for (int ib = ix; ib < nb; ib += 4) {
+
+ float sumy = 0.f;
+ for (int i = 0; i < 8; ++i) {
+ yl[i+ 0] = y4[i+ 0]; sumy += yl[i+ 0];
+ yl[i+ 8] = y4[i+32]; sumy += yl[i+ 8];
+ yl[i+16] = y4[i+64]; sumy += yl[i+16];
+ yl[i+24] = y4[i+96]; sumy += yl[i+24];
+ }
+
+ device const half * dh = &x[ib].d;
+ device const uint16_t * qs = (device const uint16_t *) x[ib].qs + 16 * iq + 4 * ir;
+
+ for (int row = 0; row < N_R0_TQ2_0; row++) {
+
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; i += 2) {
+ acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);
+ acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);
+ acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);
+ acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);
+ acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);
+ acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);
+ acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);
+ acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);
+ }
+ float dall = dh[0];
+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * 1.f/ 1.f +
+ (acc1[1] + 1.f/256.f * acc2[1]) * 1.f/ 4.f +
+ (acc1[2] + 1.f/256.f * acc2[2]) * 1.f/16.f +
+ (acc1[3] + 1.f/256.f * acc2[3]) * 1.f/64.f - sumy);
+
+ qs += step;
+ dh += step;
+ }
+
+ y4 += 4 * QK_K;
+ }
+
+ device float * dst_f32 = (device float *) dst + (uint64_t) im*args.ne0*args.ne1 + (uint64_t) r1*args.ne0;
+
+ for (int row = 0; row < N_R0_TQ2_0 && first_row + row < args.ne0; ++row) {
+ all_sum = simd_sum(sumf[row]);
+ if (tiisg == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+}
+
+[[host_name("kernel_mul_mv_tq2_0_f32")]]
+kernel void kernel_mul_mv_tq2_0_f32(
+ constant ggml_metal_kargs_mul_mv & args,
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+
+ kernel_mul_mv_tq2_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
+}
+
template
void dequantize_q4_0_t4(device const block_q4_0 * xb, short il, thread type4 & reg) {
device const uint16_t * qs = ((device const uint16_t *)xb + 1);
@@ -1098,6 +1218,162 @@ template [[host_name("kernel_repeat_f16")]] kernel kernel_repeat_t kernel_repeat
template [[host_name("kernel_repeat_i32")]] kernel kernel_repeat_t kernel_repeat;
template [[host_name("kernel_repeat_i16")]] kernel kernel_repeat_t kernel_repeat;
+template
+kernel void kernel_out_prod_impl(
+ constant ggml_metal_kargs_out_prod & args,
+ device const char * src0,
+ device const char * src1,
+ device char * dst,
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort3 tpitg[[thread_position_in_threadgroup]],
+ ushort3 ntg[[threads_per_threadgroup]]) {
+ const int i3 = tgpig.z;
+ const int i2 = tgpig.y;
+ const int i1 = tgpig.x;
+
+ const int dps2 = args.ne02 > 0 ? args.ne2 / args.ne02 : 1;
+ const int dps3 = args.ne03 > 0 ? args.ne3 / args.ne03 : 1;
+
+ const int i02 = args.ne02 > 0 ? i2 / dps2 : 0;
+ const int i03 = args.ne03 > 0 ? i3 / dps3 : 0;
+
+ device const char * src0_base = src0 + i02*args.nb02 + i03*args.nb03;
+ device const char * src1_base = src1 + i1*args.nb10 + i2*args.nb12 + i3*args.nb13;
+ device char * dst_base = dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3;
+
+ for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {
+ float acc = 0.0f;
+
+ for (int i01 = 0; i01 < args.ne01; ++i01) {
+ device const char * src0_row = src0_base + i01*args.nb01;
+ const float v0 = (float) *((device const src0_t *)(src0_row + i0*args.nb00));
+ const float v1 = (float) *((device const src1_t *)(src1_base + i01*args.nb11));
+
+ acc += v0 * v1;
+ }
+
+ *((device float *)(dst_base + i0*args.nb0)) = acc;
+ }
+}
+
+typedef decltype(kernel_out_prod_impl) kernel_out_prod_f32_t;
+typedef decltype(kernel_out_prod_impl) kernel_out_prod_f16_f32_t;
+typedef decltype(kernel_out_prod_impl) kernel_out_prod_f32_f16_t;
+typedef decltype(kernel_out_prod_impl) kernel_out_prod_f16_t;
+
+template [[host_name("kernel_out_prod_f32")]] kernel kernel_out_prod_f32_t kernel_out_prod_impl;
+template [[host_name("kernel_out_prod_f16_f32")]] kernel kernel_out_prod_f16_f32_t kernel_out_prod_impl;
+template [[host_name("kernel_out_prod_f32_f16")]] kernel kernel_out_prod_f32_f16_t kernel_out_prod_impl;
+template [[host_name("kernel_out_prod_f16")]] kernel kernel_out_prod_f16_t kernel_out_prod_impl;
+
+template