From 3447b28bbd7ab9b073d8fb25961831b272fc921f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 24 Jun 2026 17:17:50 +0000 Subject: [PATCH 1/5] feat(vllm): macOS/Metal support via vllm-metal (MLX) Add an additive Apple-Silicon path to the existing vllm Python backend so vLLM runs on macOS via vllm-metal (github.com/vllm-project/vllm-metal). Spike outcome (proven on a real M4 / macOS 26.5, Qwen3-0.6B): - vllm-metal registers through vLLM's platform-plugin entry point (metal -> vllm_metal:register); MetalPlatform activates and runs on the GPU through MLX. - LocalAI's backend.py is UNCHANGED: AsyncEngineArgs(...) -> AsyncLLMEngine.from_engine_args transparently resolves to vLLM 0.23's v1 AsyncLLM MLX engine, and async generate produced correct output. - backend.py is NOT touched: its only empty_cache() call is CUDA-only (guarded by torch.cuda.is_available()), so the benign shutdown-only "Allocator for mps is not a DeviceAllocator" noise comes from vLLM's internal EngineCore teardown, not from our code. Changes (all gated behind a darwin condition; Linux/CUDA/ROCm/Intel paths are byte-for-byte unchanged): - install.sh: darwin branch forces PYTHON_VERSION=3.12 (vllm-metal requirement), creates/activates LocalAI's managed venv via ensureVenv, then reproduces vllm-metal's installer INTO that venv (build vLLM 0.23.0 from the release source tarball against requirements/cpu.txt, then install the prebuilt vllm-metal wheel from its latest GitHub release), and runs runProtogen. installRequirements is skipped on darwin. - backend-matrix.yml: add a vllm includeDarwin entry (mps, python). - index.yaml: add metal capability + concrete metal-vllm / metal-vllm-development child entries mirroring the metal-kitten-tts template. Version coupling: vllm-metal pins vLLM 0.23.0, equal to LocalAI's current vllm pin. Bumping vllm must be coordinated with a supporting vllm-metal release; documented in install.sh and requirements-cublas13-after.txt. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- .github/backend-matrix.yml | 7 ++ backend/index.yaml | 12 +++ backend/python/vllm/install.sh | 85 ++++++++++++++++++- .../vllm/requirements-cublas13-after.txt | 3 + 4 files changed, 106 insertions(+), 1 deletion(-) diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 593e44cde714..1087b9030044 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -4974,6 +4974,13 @@ includeDarwin: - backend: "kitten-tts" tag-suffix: "-metal-darwin-arm64-kitten-tts" build-type: "mps" + # vLLM on Apple Silicon via vllm-metal (MLX). The install is custom + # (backend/python/vllm/install.sh has a darwin branch); lang stays python so + # backend_build_darwin.yml drives it through build-darwin-python-backend -> + # scripts/build/python-darwin.sh, which runs the backend's install.sh. + - backend: "vllm" + tag-suffix: "-metal-darwin-arm64-vllm" + build-type: "mps" - backend: "piper" tag-suffix: "-metal-darwin-arm64-piper" build-type: "metal" diff --git a/backend/index.yaml b/backend/index.yaml index 3f61f7b4ee1b..38d443e169a1 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -645,6 +645,7 @@ nvidia-cuda-13: "cuda13-vllm" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vllm" cpu: "cpu-vllm" + metal: "metal-vllm" - &sglang name: "sglang" license: apache-2.0 @@ -2927,6 +2928,17 @@ nvidia-cuda-13: "cuda13-vllm-development" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vllm-development" cpu: "cpu-vllm-development" + metal: "metal-vllm-development" +- !!merge <<: *vllm + name: "metal-vllm" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-vllm" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-vllm +- !!merge <<: *vllm + name: "metal-vllm-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-vllm" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-vllm - !!merge <<: *vllm name: "cuda12-vllm" uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-vllm" diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index 320ef67725a6..2b2e74c363cd 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -43,6 +43,24 @@ if [ "x${BUILD_PROFILE}" == "xcublas13" ]; then EXTRA_PIP_INSTALL_FLAGS+=" --index-strategy=unsafe-best-match" fi +# Apple Silicon (Metal/MLX) via vllm-metal. +# vllm-metal (github.com/vllm-project/vllm-metal) brings vLLM to macOS on Apple +# Silicon: it registers through vLLM's platform-plugin entry point +# (metal -> vllm_metal:register), MetalPlatform activates, and the vLLM v1 +# AsyncLLM engine runs on the GPU through MLX. LocalAI's backend.py is UNCHANGED +# on darwin — AsyncEngineArgs(...) -> AsyncLLMEngine.from_engine_args transparently +# resolves to the MLX engine (proven on a real M4 / macOS 26.5 against Qwen3-0.6B). +# +# vllm-metal REQUIRES Python 3.12, so force the portable CPython before the venv +# is created (ensureVenv reads PYTHON_VERSION/PYTHON_PATCH/PY_STANDALONE_TAG). +# The patch + standalone tag mirror the l4t13 cp312 pin — a known-good +# python-build-standalone release that also ships an aarch64-apple-darwin asset. +if [ "$(uname -s)" = "Darwin" ]; then + PYTHON_VERSION="3.12" + PYTHON_PATCH="12" + PY_STANDALONE_TAG="20251120" +fi + # JetPack 7 / L4T arm64 vllm + torch wheels come straight from PyPI now # (torch 2.11+ ships aarch64 + cu130 manylinux wheels and vllm 0.20+ ships # an aarch64 wheel pinned to that torch). They're cp312-only, so bump the @@ -57,11 +75,76 @@ if [ "x${BUILD_PROFILE}" == "xl4t13" ]; then PY_STANDALONE_TAG="20251120" fi +# ===================== Apple Silicon (Metal/MLX) ===================== +# Reproduce vllm-metal's upstream installer +# (curl -fsSL https://raw.githubusercontent.com/vllm-project/vllm-metal/main/install.sh) +# but INTO LocalAI's managed venv (ensureVenv) instead of a throwaway +# ~/.venv-vllm-metal, so the backend integrates with LocalAI's venv lifecycle +# (portable CPython, _makeVenvPortable relocation, runtime activation). The +# normal CUDA/CPU installRequirements is skipped on darwin — there is no +# macOS/arm64 vLLM wheel on PyPI; vLLM is built from source and the MLX engine +# is layered on by the vllm-metal wheel. +if [ "$(uname -s)" = "Darwin" ]; then + # Create/activate the portable 3.12 venv. On darwin USE_PIP=true and + # PORTABLE_PYTHON=true (set by scripts/build/python-darwin.sh), so this is a + # `python -m venv` based, relocatable venv. + ensureVenv + + # vllm-metal's installer drives everything through `uv`: building vLLM from + # the CPU requirements needs `--index-strategy unsafe-best-match` (mixes the + # pytorch CPU channel with PyPI), a flag plain pip does not have. The darwin + # venv is pip-based, so bootstrap uv into it. uv honours $VIRTUAL_ENV (set by + # libbackend's _activateVenv) and installs into THIS venv — same pattern the + # intel branch below relies on. + pip install uv + + # VERSION COUPLING (read before bumping vLLM!): vllm-metal pins this exact + # vLLM version and builds against its source tarball. It equals LocalAI's + # current vllm pin (see requirements-cublas13-after.txt: vllm==0.23.0). A + # vLLM bump on Linux MUST be coordinated with a vllm-metal release that + # supports the new version, or darwin builds will break. + VLLM_VERSION="0.23.0" + + _vllm_src=$(mktemp -d) + trap 'rm -rf "${_vllm_src}"' EXIT + pushd "${_vllm_src}" + # 1) Build vLLM ${VLLM_VERSION} from the release source tarball against + # the CPU requirements. vllm-metal layers its MLX platform plugin on + # top of this exact build. + curl -fsSL -o "vllm-${VLLM_VERSION}.tar.gz" \ + "https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}.tar.gz" + tar -xzf "vllm-${VLLM_VERSION}.tar.gz" + pushd "vllm-${VLLM_VERSION}" + uv pip install -r requirements/cpu.txt --index-strategy unsafe-best-match + # -Wno-parentheses: clang on macOS treats one of vLLM's C++ warnings + # as an error without it (matches the upstream installer's CXXFLAGS). + CXXFLAGS="-Wno-parentheses" uv pip install . + popd + popd + + # 2) Install the prebuilt vllm-metal wheel from its latest GitHub release. + # It pulls mlx / mlx-metal as deps and registers the `metal` platform + # plugin that backend.py resolves to at engine-init time. + _metal_wheel_url=$(curl -fsSL https://api.github.com/repos/vllm-project/vllm-metal/releases/latest \ + | grep -oE '"browser_download_url"[[:space:]]*:[[:space:]]*"[^"]+\.whl"' \ + | head -n1 | sed -E 's/.*"(https[^"]+)".*/\1/') + if [ -z "${_metal_wheel_url}" ]; then + echo "ERROR: could not resolve a vllm-metal wheel URL from the latest GitHub release" >&2 + exit 1 + fi + echo "Installing vllm-metal wheel: ${_metal_wheel_url}" + uv pip install "${_metal_wheel_url}" + + # Generate the gRPC stubs (backend_pb2*). installRequirements normally does + # this via runProtogen at the end; we skipped installRequirements on darwin, + # so call it explicitly here. + runProtogen + # Intel XPU has no upstream-published vllm wheels, so we always build vllm # from source against torch-xpu and replace the default triton with # triton-xpu (matching torch 2.11). Mirrors the upstream procedure: # https://github.com/vllm-project/vllm/blob/main/docs/getting_started/installation/gpu.xpu.inc.md -if [ "x${BUILD_TYPE}" == "xintel" ]; then +elif [ "x${BUILD_TYPE}" == "xintel" ]; then # Hide requirements-intel-after.txt so installRequirements doesn't # try `pip install vllm` (would either fail or grab a non-XPU wheel). _intel_after="${backend_dir}/requirements-intel-after.txt" diff --git a/backend/python/vllm/requirements-cublas13-after.txt b/backend/python/vllm/requirements-cublas13-after.txt index 62c486139db4..c04a25ab1d5a 100644 --- a/backend/python/vllm/requirements-cublas13-after.txt +++ b/backend/python/vllm/requirements-cublas13-after.txt @@ -4,4 +4,7 @@ # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match # so uv consults this index alongside PyPI. --extra-index-url https://wheels.vllm.ai/0.23.0/cu130 +# VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh), +# which pins this exact vLLM version. Bumping vllm here means coordinating with a +# vllm-metal release that supports the new version, or macOS/Metal builds break. vllm==0.23.0 From 7743a0abc0cbf4007c8e315b13070443f0920b13 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 24 Jun 2026 20:03:14 +0000 Subject: [PATCH 2/5] chore(vllm): track the darwin vllm-metal pin via the autobumper The Apple Silicon build pinned vLLM 0.23.0 as a hidden string in install.sh while floating the vllm-metal wheel on releases/latest - the two could drift apart silently. Make both a tracked, reproducible pair (VLLM_METAL_VERSION + VLLM_VERSION), fetch the wheel by tag, and add .github/bump_vllm_metal.sh wired into bump_deps.yaml. It tracks vllm-project/vllm-metal (not vllm/vllm latest), reading the coupled vLLM source version from vllm-metal's own installer, and opens a bump PR - mirroring the existing bump_vllm_wheel.sh for the cu130 wheel. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- .github/bump_vllm_metal.sh | 57 ++++++++++++++++++++++++++++++++ .github/workflows/bump_deps.yaml | 36 ++++++++++++++++++++ backend/python/vllm/install.sh | 27 +++++++++------ 3 files changed, 110 insertions(+), 10 deletions(-) create mode 100755 .github/bump_vllm_metal.sh diff --git a/.github/bump_vllm_metal.sh b/.github/bump_vllm_metal.sh new file mode 100755 index 000000000000..249e39b4a56e --- /dev/null +++ b/.github/bump_vllm_metal.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Bump the vllm-metal pins in the vLLM backend's darwin (Apple Silicon) install +# path. The macOS/Metal build (backend/python/vllm/install.sh, Darwin branch) +# installs vllm-metal, which is version-locked to a specific vLLM source release. +# Two values must move together: +# VLLM_METAL_VERSION -> the vllm-metal GitHub release tag (its prebuilt wheel) +# VLLM_VERSION -> the vLLM source version that release builds against +# vllm-metal declares the latter in its OWN install.sh as `vllm_v="X.Y.Z"`. This +# script reads both from vllm-metal's latest release and rewrites them atomically +# -- mirroring bump_vllm_wheel.sh, which does the same for the Linux cu130 wheel. +# +# This deliberately tracks vllm-project/vllm-metal, NOT vllm-project/vllm: the +# darwin build can only use the exact vLLM version vllm-metal supports, so it may +# lag the Linux pin (requirements-cublas13-after.txt) until vllm-metal catches up. +set -xe +REPO=$1 # vllm-project/vllm-metal +FILE=$2 # backend/python/vllm/install.sh +VAR=$3 # VLLM_METAL_VERSION (used for the workflow's output file names) + +if [ -z "$FILE" ] || [ -z "$REPO" ] || [ -z "$VAR" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# vllm-metal ships frequent dev releases, all flagged as non-prerelease, so +# /releases/latest returns the newest one (with its cp312 wheel asset). +LATEST_TAG=$(curl -sS -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$REPO/releases/latest" \ + | python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'])") + +# The coupled vLLM source version lives in vllm-metal's installer at that tag. +NEW_VLLM_VERSION=$(curl -fsSL \ + "https://raw.githubusercontent.com/$REPO/$LATEST_TAG/install.sh" \ + | grep -oE 'vllm_v="[0-9]+\.[0-9]+\.[0-9]+"' | head -1 | cut -d'"' -f2) + +if [ -z "$LATEST_TAG" ] || [ -z "$NEW_VLLM_VERSION" ]; then + echo "Could not resolve vllm-metal tag ($LATEST_TAG) or its vllm_v ($NEW_VLLM_VERSION)." >&2 + exit 1 +fi + +set +e +CURRENT_TAG=$(grep -oE 'VLLM_METAL_VERSION="[^"]*"' "$FILE" | head -1 | cut -d'"' -f2) +set -e + +# Rewrite both pins. peter-evans/create-pull-request opens no PR on a clean tree, +# so a no-op rewrite (already current) is safe. +sed -i "$FILE" \ + -e "s|VLLM_METAL_VERSION=\"[^\"]*\"|VLLM_METAL_VERSION=\"$LATEST_TAG\"|" \ + -e "s|VLLM_VERSION=\"[^\"]*\"|VLLM_VERSION=\"$NEW_VLLM_VERSION\"|" + +if [ -z "$CURRENT_TAG" ]; then + echo "Could not find VLLM_METAL_VERSION=\"...\" in $FILE." >&2 + exit 0 +fi + +echo "vllm-metal ${CURRENT_TAG} -> ${LATEST_TAG} (builds vLLM ${NEW_VLLM_VERSION}): https://github.com/$REPO/releases/tag/${LATEST_TAG}" >> "${VAR}_message.txt" +echo "${LATEST_TAG}" >> "${VAR}_commit.txt" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index aa4b21af7a9a..a2c37881f83d 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -154,3 +154,39 @@ jobs: branch: "update/VLLM_VERSION" body: ${{ steps.bump.outputs.message }} signoff: true + + bump-vllm-metal: + # The darwin (Apple Silicon) vLLM build installs vllm-metal, which is locked + # to a specific vLLM source release. install.sh pins both VLLM_METAL_VERSION + # (the wheel release) and VLLM_VERSION (the vLLM it builds against); this job + # tracks vllm-project/vllm-metal and rewrites both atomically. Separate from + # bump-vllm-wheel because darwin follows vllm-metal, not vllm/vllm latest. + if: github.repository == 'mudler/LocalAI' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Bump vllm-metal pin 🔧 + id: bump + run: | + bash .github/bump_vllm_metal.sh vllm-project/vllm-metal backend/python/vllm/install.sh VLLM_METAL_VERSION + { + echo 'message<> "$GITHUB_OUTPUT" + { + echo 'commit<> "$GITHUB_OUTPUT" + rm -rfv VLLM_METAL_VERSION_message.txt VLLM_METAL_VERSION_commit.txt + - name: Create Pull Request + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ secrets.UPDATE_BOT_TOKEN }} + push-to-fork: ci-forks/LocalAI + commit-message: ':arrow_up: Update vllm-project/vllm-metal (darwin)' + title: 'chore: :arrow_up: Update vllm-metal (darwin) to `${{ steps.bump.outputs.commit }}`' + branch: "update/VLLM_METAL_VERSION" + body: ${{ steps.bump.outputs.message }} + signoff: true diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index 2b2e74c363cd..5e4feb4fb66d 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -98,11 +98,15 @@ if [ "$(uname -s)" = "Darwin" ]; then # intel branch below relies on. pip install uv - # VERSION COUPLING (read before bumping vLLM!): vllm-metal pins this exact - # vLLM version and builds against its source tarball. It equals LocalAI's - # current vllm pin (see requirements-cublas13-after.txt: vllm==0.23.0). A - # vLLM bump on Linux MUST be coordinated with a vllm-metal release that - # supports the new version, or darwin builds will break. + # vllm-metal version pins -- AUTO-BUMPED by .github/bump_vllm_metal.sh, which + # tracks vllm-project/vllm-metal releases (NOT vllm/vllm latest). VLLM_METAL_VERSION + # is the vllm-metal release tag (its prebuilt wheel); VLLM_VERSION is the vLLM + # source version that release builds against (vllm-metal declares it as vllm_v=). + # They move in lockstep, so darwin can lag the Linux vllm pin + # (requirements-cublas13-after.txt, bumped independently against vllm/vllm) until + # vllm-metal supports a newer vLLM. Keep both as plain double-quoted assignments + # each on their own line so the bumper's sed can rewrite them. + VLLM_METAL_VERSION="v0.3.0.dev20260622062346" VLLM_VERSION="0.23.0" _vllm_src=$(mktemp -d) @@ -122,14 +126,17 @@ if [ "$(uname -s)" = "Darwin" ]; then popd popd - # 2) Install the prebuilt vllm-metal wheel from its latest GitHub release. - # It pulls mlx / mlx-metal as deps and registers the `metal` platform - # plugin that backend.py resolves to at engine-init time. - _metal_wheel_url=$(curl -fsSL https://api.github.com/repos/vllm-project/vllm-metal/releases/latest \ + # 2) Install the prebuilt vllm-metal wheel from the PINNED release + # (${VLLM_METAL_VERSION}). It pulls mlx / mlx-metal as deps and registers + # the `metal` platform plugin that backend.py resolves to at engine-init + # time. Pinning the tag (vs releases/latest) keeps the wheel and the vLLM + # source build above reproducible and coupled; .github/bump_vllm_metal.sh + # advances both together. + _metal_wheel_url=$(curl -fsSL "https://api.github.com/repos/vllm-project/vllm-metal/releases/tags/${VLLM_METAL_VERSION}" \ | grep -oE '"browser_download_url"[[:space:]]*:[[:space:]]*"[^"]+\.whl"' \ | head -n1 | sed -E 's/.*"(https[^"]+)".*/\1/') if [ -z "${_metal_wheel_url}" ]; then - echo "ERROR: could not resolve a vllm-metal wheel URL from the latest GitHub release" >&2 + echo "ERROR: could not resolve a vllm-metal wheel URL for release ${VLLM_METAL_VERSION}" >&2 exit 1 fi echo "Installing vllm-metal wheel: ${_metal_wheel_url}" From af7d0e8b4062c593912c898a66d2a6365bd096cc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 24 Jun 2026 20:25:07 +0000 Subject: [PATCH 3/5] chore(vllm): derive the darwin vLLM version, drop the second pin Follow-up: VLLM_VERSION was still a hardcoded string duplicating what VLLM_METAL_VERSION already determines. Derive it at install time from vllm-metal's own installer (vllm_v=) at the pinned tag - one source of truth, no second value to drift. The bumper now touches only VLLM_METAL_VERSION; the derivation is immutable per tag, so builds stay reproducible. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- .github/bump_vllm_metal.sh | 24 +++++++++++------------- backend/python/vllm/install.sh | 27 ++++++++++++++++++--------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/.github/bump_vllm_metal.sh b/.github/bump_vllm_metal.sh index 249e39b4a56e..f842680d51fd 100755 --- a/.github/bump_vllm_metal.sh +++ b/.github/bump_vllm_metal.sh @@ -1,13 +1,11 @@ #!/bin/bash -# Bump the vllm-metal pins in the vLLM backend's darwin (Apple Silicon) install -# path. The macOS/Metal build (backend/python/vllm/install.sh, Darwin branch) -# installs vllm-metal, which is version-locked to a specific vLLM source release. -# Two values must move together: -# VLLM_METAL_VERSION -> the vllm-metal GitHub release tag (its prebuilt wheel) -# VLLM_VERSION -> the vLLM source version that release builds against -# vllm-metal declares the latter in its OWN install.sh as `vllm_v="X.Y.Z"`. This -# script reads both from vllm-metal's latest release and rewrites them atomically -# -- mirroring bump_vllm_wheel.sh, which does the same for the Linux cu130 wheel. +# Bump the single vllm-metal pin (VLLM_METAL_VERSION) in the vLLM backend's +# darwin (Apple Silicon) install path. The macOS/Metal build +# (backend/python/vllm/install.sh, Darwin branch) installs vllm-metal, which is +# version-locked to a specific vLLM source release. install.sh derives that vLLM +# version at build time from vllm-metal's own installer (`vllm_v=`) at the pinned +# tag, so there is only ONE value to bump here -- mirroring bump_vllm_wheel.sh, +# which bumps the Linux cu130 wheel pin. # # This deliberately tracks vllm-project/vllm-metal, NOT vllm-project/vllm: the # darwin build can only use the exact vLLM version vllm-metal supports, so it may @@ -42,11 +40,11 @@ set +e CURRENT_TAG=$(grep -oE 'VLLM_METAL_VERSION="[^"]*"' "$FILE" | head -1 | cut -d'"' -f2) set -e -# Rewrite both pins. peter-evans/create-pull-request opens no PR on a clean tree, -# so a no-op rewrite (already current) is safe. +# Rewrite the single pin. install.sh derives VLLM_VERSION from this tag at build +# time, so there is nothing else to touch. peter-evans/create-pull-request opens +# no PR on a clean tree, so a no-op rewrite (already current) is safe. sed -i "$FILE" \ - -e "s|VLLM_METAL_VERSION=\"[^\"]*\"|VLLM_METAL_VERSION=\"$LATEST_TAG\"|" \ - -e "s|VLLM_VERSION=\"[^\"]*\"|VLLM_VERSION=\"$NEW_VLLM_VERSION\"|" + -e "s|VLLM_METAL_VERSION=\"[^\"]*\"|VLLM_METAL_VERSION=\"$LATEST_TAG\"|" if [ -z "$CURRENT_TAG" ]; then echo "Could not find VLLM_METAL_VERSION=\"...\" in $FILE." >&2 diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index 5e4feb4fb66d..c0357f436f37 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -98,16 +98,25 @@ if [ "$(uname -s)" = "Darwin" ]; then # intel branch below relies on. pip install uv - # vllm-metal version pins -- AUTO-BUMPED by .github/bump_vllm_metal.sh, which - # tracks vllm-project/vllm-metal releases (NOT vllm/vllm latest). VLLM_METAL_VERSION - # is the vllm-metal release tag (its prebuilt wheel); VLLM_VERSION is the vLLM - # source version that release builds against (vllm-metal declares it as vllm_v=). - # They move in lockstep, so darwin can lag the Linux vllm pin - # (requirements-cublas13-after.txt, bumped independently against vllm/vllm) until - # vllm-metal supports a newer vLLM. Keep both as plain double-quoted assignments - # each on their own line so the bumper's sed can rewrite them. + # The ONLY darwin version pin -- AUTO-BUMPED by .github/bump_vllm_metal.sh, + # which tracks vllm-project/vllm-metal releases (NOT vllm/vllm latest). Keep + # it as a plain double-quoted assignment on its own line so the bumper's sed + # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux + # vllm pin (requirements-cublas13-after.txt, bumped independently against + # vllm/vllm) until vllm-metal supports a newer vLLM. VLLM_METAL_VERSION="v0.3.0.dev20260622062346" - VLLM_VERSION="0.23.0" + + # The coupled vLLM source version is whatever this vllm-metal release builds + # against -- it declares it in its own installer as `vllm_v=`. Derive it from + # the PINNED tag rather than hardcoding a second value that could drift. The + # tag is immutable, so this stays reproducible across rebuilds. + VLLM_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/vllm-project/vllm-metal/${VLLM_METAL_VERSION}/install.sh" \ + | grep -oE 'vllm_v="[0-9]+\.[0-9]+\.[0-9]+"' | head -n1 | cut -d'"' -f2) + if [ -z "${VLLM_VERSION}" ]; then + echo "ERROR: could not derive the vLLM version from vllm-metal ${VLLM_METAL_VERSION}" >&2 + exit 1 + fi + echo "vllm-metal ${VLLM_METAL_VERSION} builds against vLLM ${VLLM_VERSION}" _vllm_src=$(mktemp -d) trap 'rm -rf "${_vllm_src}"' EXIT From bfb9a40d5844e1118be5ec672dec53d243269c00 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 24 Jun 2026 20:54:30 +0000 Subject: [PATCH 4/5] fix(vllm): fetch the vllm-metal wheel without the GitHub API The darwin build resolved the wheel URL via api.github.com, whose unauthenticated rate limit (60/hr per IP) 403s on shared macOS runners (observed after the 9-min vLLM source build). Construct the release-asset download URL deterministically from the pinned tag and the cp312/arm64 wheel name instead - no API call, no rate limit. Verified the URL resolves (200). Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- backend/python/vllm/install.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index c0357f436f37..85c1e97b0343 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -135,19 +135,14 @@ if [ "$(uname -s)" = "Darwin" ]; then popd popd - # 2) Install the prebuilt vllm-metal wheel from the PINNED release - # (${VLLM_METAL_VERSION}). It pulls mlx / mlx-metal as deps and registers - # the `metal` platform plugin that backend.py resolves to at engine-init - # time. Pinning the tag (vs releases/latest) keeps the wheel and the vLLM - # source build above reproducible and coupled; .github/bump_vllm_metal.sh - # advances both together. - _metal_wheel_url=$(curl -fsSL "https://api.github.com/repos/vllm-project/vllm-metal/releases/tags/${VLLM_METAL_VERSION}" \ - | grep -oE '"browser_download_url"[[:space:]]*:[[:space:]]*"[^"]+\.whl"' \ - | head -n1 | sed -E 's/.*"(https[^"]+)".*/\1/') - if [ -z "${_metal_wheel_url}" ]; then - echo "ERROR: could not resolve a vllm-metal wheel URL for release ${VLLM_METAL_VERSION}" >&2 - exit 1 - fi + # 2) Install the prebuilt vllm-metal wheel for the PINNED release. It pulls + # mlx / mlx-metal as deps and registers the `metal` platform plugin that + # backend.py resolves to at engine-init time. Build the release-asset URL + # deterministically (tag + the cp312/arm64 wheel name) rather than querying + # api.github.com, whose unauthenticated rate limit (60/hr per IP) 403s on + # shared CI runners. The wheel version is the tag without its leading 'v'. + _metal_wheel="vllm_metal-${VLLM_METAL_VERSION#v}-cp312-cp312-macosx_11_0_arm64.whl" + _metal_wheel_url="https://github.com/vllm-project/vllm-metal/releases/download/${VLLM_METAL_VERSION}/${_metal_wheel}" echo "Installing vllm-metal wheel: ${_metal_wheel_url}" uv pip install "${_metal_wheel_url}" From 5e3774dfe313bebdb910f9e6c7954caf773e8191 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 24 Jun 2026 21:31:41 +0000 Subject: [PATCH 5/5] fix(vllm): fail Score cleanly when the engine returns no prompt_logprobs Audit of the Score path against vllm-metal (MLX on macOS): the engine accepts SamplingParams(prompt_logprobs=1) but returns an all-None prompt_logprobs list rather than computing it, so scoring is not supported there. The old guard treated the truthy [None] list as valid and silently scored every candidate as 0. Detect the all-None case and return UNIMPLEMENTED instead. No-op on Linux/CUDA, which populate real entries. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:opus-4.8 [Claude Code] --- backend/python/vllm/backend.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index a3884913711f..1e93f26e2324 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -457,9 +457,14 @@ async def Score(self, request, context): except Exception: pass - if last_output is None or not getattr(last_output, "prompt_logprobs", None): - context.set_code(grpc.StatusCode.INTERNAL) - context.set_details("vLLM did not return prompt_logprobs") + _pl = getattr(last_output, "prompt_logprobs", None) if last_output is not None else None + # Some engines accept the prompt_logprobs request but return a + # list of all-None entries instead of computing them (observed + # with vllm-metal's MLX backend on macOS). Treat that as + # unsupported rather than silently scoring every candidate as 0. + if not _pl or all(e is None for e in _pl): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("This backend did not return prompt_logprobs; scoring is unsupported on this engine (e.g. vllm-metal / MLX on macOS).") return backend_pb2.ScoreResponse() prompt_logprobs = last_output.prompt_logprobs