From 5f515eec6d9b25ef8556daa383ae0bed1b07324c Mon Sep 17 00:00:00 2001 From: abetlen Date: Fri, 5 Jun 2026 23:57:41 -0700 Subject: [PATCH 1/2] docs: add Gemma 4 QAT Colab notebook --- README.md | 2 + examples/colab/gemma4_qat_notebook.ipynb | 162 +++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 examples/colab/gemma4_qat_notebook.ipynb diff --git a/README.md b/README.md index 8f7b65e83..b63fbec37 100644 --- a/README.md +++ b/README.md @@ -540,6 +540,8 @@ Below are the supported multi-modal models and their respective chat handlers (P Try Gemma 4 12B in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb) +Try Gemma 4 12B QAT in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/gemma4_qat_notebook.ipynb) + Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. ```python diff --git a/examples/colab/gemma4_qat_notebook.ipynb b/examples/colab/gemma4_qat_notebook.ipynb new file mode 100644 index 000000000..2cced5a89 --- /dev/null +++ b/examples/colab/gemma4_qat_notebook.ipynb @@ -0,0 +1,162 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "accelerator": "GPU", + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gemma 4 12B QAT Multimodal Chat\n", + "\n", + "Run the Gemma 4 12B QAT GGUF model locally in Google Colab with the pre-built CUDA wheel for `llama-cpp-python`.\n", + "\n", + "Use a GPU runtime before running this notebook: **Runtime > Change runtime type > T4 GPU**.\n", + "\n", + "Current Colab CUDA images commonly provide CUDA 12 user-space libraries even when `nvidia-smi` reports a CUDA 13-capable driver, so this notebook installs the `cu125` wheel. If your runtime provides `libcudart.so.13`, switch the wheel index URL to `/whl/cu130`.\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install --no-cache-dir --upgrade --force-reinstall \\\n", + " \"huggingface-hub>=0.23.0\" \\\n", + " llama-cpp-python \\\n", + " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu125\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from llama_cpp import Llama\n", + "from llama_cpp.llama_chat_format import Gemma4ChatHandler\n", + "\n", + "MODEL_REPO = \"unsloth/gemma-4-12B-it-qat-GGUF\"\n", + "MODEL_FILE = \"gemma-4-12B-it-qat-UD-Q4_K_XL.gguf\"\n", + "MMPROJ_FILE = \"mmproj-F16.gguf\"\n", + "\n", + "chat_handler = Gemma4ChatHandler.from_pretrained(\n", + " repo_id=MODEL_REPO,\n", + " filename=MMPROJ_FILE,\n", + " verbose=False,\n", + ")\n", + "\n", + "llm = Llama.from_pretrained(\n", + " repo_id=MODEL_REPO,\n", + " filename=MODEL_FILE,\n", + " chat_handler=chat_handler,\n", + " n_gpu_layers=-1,\n", + " n_ctx=8192,\n", + " flash_attn=True,\n", + " verbose=False,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Image, display\n", + "\n", + "IMAGE_URL = \"https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/mtmd/test-1.jpeg\"\n", + "\n", + "display(Image(url=IMAGE_URL, width=320))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = llm.create_chat_completion(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"Describe this image in one concise sentence.\"},\n", + " {\"type\": \"image_url\", \"image_url\": {\"url\": IMAGE_URL}},\n", + " ],\n", + " }\n", + " ],\n", + " max_tokens=128,\n", + " temperature=0.2,\n", + ")\n", + "\n", + "print(response[\"choices\"][0][\"message\"][\"content\"])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tools = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"record_image_observation\",\n", + " \"description\": \"Record a structured observation about an image.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"main_subject\": {\"type\": \"string\"},\n", + " \"setting\": {\"type\": \"string\"},\n", + " \"notable_details\": {\n", + " \"type\": \"array\",\n", + " \"items\": {\"type\": \"string\"},\n", + " },\n", + " \"confidence\": {\"type\": \"number\", \"minimum\": 0, \"maximum\": 1},\n", + " },\n", + " \"required\": [\"main_subject\", \"setting\", \"notable_details\", \"confidence\"],\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "response = llm.create_chat_completion(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"Use the provided tool to record a structured observation for this image.\"},\n", + " {\"type\": \"image_url\", \"image_url\": {\"url\": IMAGE_URL}},\n", + " ],\n", + " }\n", + " ],\n", + " tools=tools,\n", + " tool_choice={\"type\": \"function\", \"function\": {\"name\": \"record_image_observation\"}},\n", + " max_tokens=256,\n", + " temperature=0.0,\n", + ")\n", + "\n", + "message = response[\"choices\"][0][\"message\"]\n", + "print(message.get(\"tool_calls\", message.get(\"content\")))\n" + ] + } + ] +} From 52658577e336d26fdf812bcc3356d89467457f7a Mon Sep 17 00:00:00 2001 From: abetlen Date: Fri, 5 Jun 2026 23:59:29 -0700 Subject: [PATCH 2/2] docs: rename Gemma 4 QAT Colab notebook --- README.md | 2 +- .../colab/{gemma4_qat_notebook.ipynb => Gemma4-12B-QAT.ipynb} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename examples/colab/{gemma4_qat_notebook.ipynb => Gemma4-12B-QAT.ipynb} (100%) diff --git a/README.md b/README.md index b63fbec37..c02df9eed 100644 --- a/README.md +++ b/README.md @@ -540,7 +540,7 @@ Below are the supported multi-modal models and their respective chat handlers (P Try Gemma 4 12B in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb) -Try Gemma 4 12B QAT in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/gemma4_qat_notebook.ipynb) +Try Gemma 4 12B QAT in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/Gemma4-12B-QAT.ipynb) Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. diff --git a/examples/colab/gemma4_qat_notebook.ipynb b/examples/colab/Gemma4-12B-QAT.ipynb similarity index 100% rename from examples/colab/gemma4_qat_notebook.ipynb rename to examples/colab/Gemma4-12B-QAT.ipynb