Fix managed llama.cpp offloading to the integrated GPU instead of the discrete NVIDIA card

src/cli/models-handlers.ts

@@ -23,6 +23,7 @@ import {
   readBackendVersion,
   removeModel,
   resolveChatTemplatePath,
+  resolveDownloadAsset,
   resolveManagedDevice,
   resolveMmprojFilePath,
   resolvePlatformAsset,
@@ -149,6 +150,28 @@ export async function runLocalModelsUse(idArg: string | undefined): Promise<numb
   return 0;
 }
 
+/**
+ * Describe the installed compute backend and flag it when this machine
+ * warrants a different one. Every Windows zip ships the same
+ * `llama-server.exe`, so without the recorded asset name there is no way
+ * to tell a Vulkan install from a CUDA one — which is exactly the state
+ * that silently leaves an NVIDIA box enumerating (and offloading to) its
+ * integrated GPU.
+ */
+function describeComputeBackend(installed: string | undefined): string {
+  if (installed === undefined) {
+    return "unknown (installed before variant tracking) — run 'models update' to refresh";
+  }
+  let want: string;
+  try {
+    want = resolveDownloadAsset().assetName;
+  } catch {
+    return installed;
+  }
+  if (installed === want) return installed;
+  return `${installed} — this machine warrants ${want}; run 'models update'`;
+}
+
 export async function runLocalModelsStatus(): Promise<number> {
   const cfg = getConfig();
   if (cfg.localModels.mode === "external") {
@@ -172,6 +195,7 @@ export async function runLocalModelsStatus(): Promise<number> {
   process.stdout.write(
     `backend:        ${ver?.tag ?? "(none)"} (installed ${ver?.downloadedAt ?? "n/a"}), binary ${binOk ? "ok" : "missing"}\n`,
   );
+  process.stdout.write(`compute:        ${describeComputeBackend(ver?.asset)}\n`);
   process.stdout.write(`active model:   ${modelLine}\n`);
   process.stdout.write(
     `daemon:         ${st.running ? `running (pid ${st.pid})` : "stopped"}  ${cfg.localModels.url}\n`,

src/local-llm/backend-installer.ts

@@ -130,8 +130,14 @@ export async function checkForBackendUpdate(
 ): Promise<{ updateAvailable: boolean; latestTag: string; currentTag: string | null }> {
   const current = readBackendVersion(dataDir);
   const release = await fetchLatestRelease();
+  // A variant mismatch counts as an update even at the same tag: a
+  // Windows box that installed the Vulkan build before its NVIDIA driver
+  // was present would otherwise keep running Vulkan (and offloading to
+  // whatever device Vulkan enumerates) forever.
+  const variantStale =
+    current?.asset !== undefined && current.asset !== resolveDownloadAsset().assetName;
   return {
-    updateAvailable: current?.tag !== release.tag,
+    updateAvailable: current?.tag !== release.tag || variantStale,
     latestTag: release.tag,
     currentTag: current?.tag ?? null,
   };
@@ -342,6 +348,7 @@ export async function downloadBackend(
   writeBackendVersion(dataDir, {
     tag: release.tag,
     downloadedAt: new Date().toISOString(),
+    asset: assetName,
   });
 
   return { ok: true, tag: release.tag };

src/local-llm/backend-version.ts

@@ -6,6 +6,16 @@ import { resolveVersionFilePath } from "./backend-paths.js";
 export interface BackendVersionInfo {
   tag: string;
   downloadedAt: string;
+  /**
+   * Release asset actually installed. On Windows the same
+   * `llama-server.exe` ships in a Vulkan and two CUDA zips, so the
+   * binary's presence alone cannot tell us which compute backend is on
+   * disk. Recording it lets `checkForBackendUpdate` offer a re-download
+   * when the machine now warrants a different variant (e.g. the NVIDIA
+   * driver was installed after the first Vulkan-only install). Absent on
+   * installs predating this field.
+   */
+  asset?: string;
 }
 
 export function readBackendVersion(dataDir: string): BackendVersionInfo | null {

src/local-llm/gpu-devices.test.ts

@@ -115,6 +115,50 @@ describe("pickBestDevice", () => {
     expect(pickBestDevice([])).toBeNull();
   });
 
+  // AMD APU iGPUs carry no "Intel"/"integrated" marker and Vulkan
+  // reports their shared-RAM heap as larger than the dGPU's VRAM, so the
+  // VRAM tiebreak used to hand the model to the iGPU.
+  it.each([
+    "AMD Radeon(TM) Graphics",
+    "AMD Radeon 780M Graphics",
+    "AMD Radeon(TM) Vega 8 Graphics",
+    "Intel(R) Graphics (RPL-S)",
+  ])("prefers the dGPU over integrated %s reporting more memory", (igpu) => {
+    const devices: GpuDevice[] = [
+      {
+        id: "Vulkan0",
+        description: igpu,
+        totalMemMiB: 16384,
+        freeMemMiB: 16384,
+      },
+      {
+        id: "Vulkan1",
+        description: "NVIDIA GeForce RTX 4060 Laptop GPU",
+        totalMemMiB: 8188,
+        freeMemMiB: 8188,
+      },
+    ];
+    expect(pickBestDevice(devices)).toBe("Vulkan1");
+  });
+
+  it("does not demote a discrete Intel Arc to integrated", () => {
+    const devices: GpuDevice[] = [
+      {
+        id: "Vulkan0",
+        description: "Intel(R) Arc(TM) A770 Graphics",
+        totalMemMiB: 16384,
+        freeMemMiB: 16384,
+      },
+      {
+        id: "Vulkan1",
+        description: "Intel(R) UHD Graphics 770",
+        totalMemMiB: 32000,
+        freeMemMiB: 32000,
+      },
+    ];
+    expect(pickBestDevice(devices)).toBe("Vulkan0");
+  });
+
   it("falls back to an integrated GPU when no discrete is present", () => {
     const devices: GpuDevice[] = [
       {

src/local-llm/gpu-devices.ts

@@ -29,11 +29,23 @@ export interface GpuDevice {
 const SOFTWARE_DEVICE_RE = /llvmpipe|lavapipe|swiftshader|software/i;
 
 /**
- * Integrated GPUs (Intel iGPU, some AMD APUs) share system RAM and are
- * far slower than a discrete card. Used as a tiebreaker so an auto-pick
- * prefers the dGPU even when the iGPU reports more "memory".
+ * Positive discrete-GPU hints, checked *before* the integrated ones so a
+ * discrete card whose name also trips an integrated pattern (Intel Arc,
+ * "Radeon RX") is never demoted.
  */
-const INTEGRATED_DEVICE_RE = /intel|integrated|\buhd\b|\biris\b/i;
+const DISCRETE_DEVICE_RE =
+  /nvidia|geforce|\brtx\b|\bgtx\b|quadro|tesla|\barc\b|radeon\s*(rx|pro)\b/i;
+
+/**
+ * Integrated GPUs (Intel iGPU, AMD APUs) share system RAM and are far
+ * slower than a discrete card. Vulkan reports their heap as a slice of
+ * system RAM — often *larger* than a dGPU's VRAM — so without this the
+ * VRAM tiebreak picks the iGPU. AMD APU marketing names carry no "APU"
+ * marker: "AMD Radeon(TM) Graphics", "AMD Radeon 780M Graphics",
+ * "Radeon(TM) Vega 8 Graphics".
+ */
+const INTEGRATED_DEVICE_RE =
+  /intel|integrated|\buhd\b|\biris\b|\bvega\b|\bapu\b|radeon.*\b\d{3}m\b|radeon(\(tm\))?\s*graphics/i;
 
 /**
  * Parse the stdout/stderr of `llama-server --list-devices`. Pure — no
@@ -72,8 +84,16 @@ export function parseListDevices(output: string): GpuDevice[] {
   return devices;
 }
 
-function isDiscrete(description: string): boolean {
-  return !INTEGRATED_DEVICE_RE.test(description);
+/**
+ * Offload desirability, higher is better: 2 = known discrete card,
+ * 1 = unrecognised (assume discrete — safer than assuming iGPU),
+ * 0 = known integrated. Discrete hints win outright so "Intel Arc" and
+ * "Radeon RX" are not caught by the integrated patterns.
+ */
+export function deviceClassRank(description: string): 0 | 1 | 2 {
+  if (DISCRETE_DEVICE_RE.test(description)) return 2;
+  if (INTEGRATED_DEVICE_RE.test(description)) return 0;
+  return 1;
 }
 
 /**
@@ -88,9 +108,8 @@ export function pickBestDevice(devices: readonly GpuDevice[]): string | null {
   );
   if (candidates.length === 0) return null;
   const ranked = [...candidates].sort((a, b) => {
-    const aDiscrete = isDiscrete(a.description);
-    const bDiscrete = isDiscrete(b.description);
-    if (aDiscrete !== bDiscrete) return aDiscrete ? -1 : 1;
+    const rank = deviceClassRank(b.description) - deviceClassRank(a.description);
+    if (rank !== 0) return rank;
     return b.totalMemMiB - a.totalMemMiB;
   });
   return ranked[0]?.id ?? null;

src/local-llm/index.ts

@@ -19,6 +19,8 @@ export {
   type PlatformAsset,
 } from "./platform-assets.js";
 
+export { resolveDownloadAsset } from "./windows-backend-variant.js";
+
 export {
   resolveBackendDir,
   resolveModelsDir,

src/local-llm/windows-backend-variant.test.ts

@@ -43,7 +43,12 @@ describe("selectWindowsBackendAsset", () => {
     expect(selectWindowsBackendAsset(null)).toBe(WINDOWS_BACKEND_ASSETS.vulkan);
   });
 
-  it("picks cuda-13.3 when the driver supports >= 13.3", () => {
+  it("picks cuda-13.3 for any 13.x driver (minor-version compatibility)", () => {
+    // Blackwell (sm_120) exists only in the 13.3 build; a 13.0 driver
+    // runs it fine, so it must not be sent to the 12.4 build.
+    expect(selectWindowsBackendAsset({ major: 13, minor: 0 })).toBe(
+      WINDOWS_BACKEND_ASSETS.cuda133,
+    );
     expect(selectWindowsBackendAsset({ major: 13, minor: 3 })).toBe(
       WINDOWS_BACKEND_ASSETS.cuda133,
     );
@@ -55,18 +60,13 @@ describe("selectWindowsBackendAsset", () => {
     );
   });
 
-  it("picks cuda-12.4 when the driver supports >= 12.4 but < 13.3", () => {
+  it("picks cuda-12.4 for a 12.x driver at or above 12.4", () => {
     expect(selectWindowsBackendAsset({ major: 12, minor: 4 })).toBe(
       WINDOWS_BACKEND_ASSETS.cuda124,
     );
     expect(selectWindowsBackendAsset({ major: 12, minor: 6 })).toBe(
       WINDOWS_BACKEND_ASSETS.cuda124,
     );
-    // 13.0 is newer than 12.4 but older than the 13.3 build we ship, so
-    // the safe choice is the 12.4 build (a 13.0 driver runs it fine).
-    expect(selectWindowsBackendAsset({ major: 13, minor: 0 })).toBe(
-      WINDOWS_BACKEND_ASSETS.cuda124,
-    );
   });
 
   it("falls back to Vulkan when the driver is older than 12.4", () => {

src/local-llm/windows-backend-variant.ts

@@ -47,7 +47,12 @@ function isAtLeast(v: CudaVersion, major: number, minor: number): boolean {
  */
 export function selectWindowsBackendAsset(cuda: CudaVersion | null): string {
   if (cuda === null) return WINDOWS_BACKEND_ASSETS.vulkan;
-  if (isAtLeast(cuda, 13, 3)) return WINDOWS_BACKEND_ASSETS.cuda133;
+  // Threshold is the driver's CUDA *major*, not the build's exact minor:
+  // CUDA minor-version compatibility lets a 13.3-built binary run on any
+  // 13.x driver. Gating on >= 13.3 sent 13.0 drivers to the 12.4 build,
+  // which has no sm_120 (Blackwell) code at all — that toolkit predates
+  // the arch, so RTX 50-series had to JIT from PTX or fall off the GPU.
+  if (isAtLeast(cuda, 13, 0)) return WINDOWS_BACKEND_ASSETS.cuda133;
   if (isAtLeast(cuda, 12, 4)) return WINDOWS_BACKEND_ASSETS.cuda124;
   return WINDOWS_BACKEND_ASSETS.vulkan;
 }