Skip to content

Commit 673a80a

Browse files
Copilotmudler
Copilot
and
mudler
authored
feat: Filter backend gallery by system capabilities (#7950)
* Initial plan * Add backend gallery filtering based on system capabilities Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Refactor L4T backend check to come before NVIDIA check Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Refactor: move capabilities business logic to capabilities.go and use constants Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * feat: display system capability in webui and refactor tests Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * chore: rename System/Capability Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * refactor: use getSystemCapabilities in IsBackendCompatible for consistency Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * refactor: keep unused constants private in capabilities.go Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * fix: skip AMD/ROCm and Intel/SYCL tests on darwin Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 2554e9f commit 673a80a

6 files changed

Lines changed: 383 additions & 16 deletions

File tree

‎core/gallery/backend_types.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,25 @@ func (m *GalleryBackend) IsMeta() bool {
6363
return len(m.CapabilitiesMap) > 0 && m.URI == ""
6464
}
6565

66+
// IsCompatibleWith checks if the backend is compatible with the current system capability.
67+
// For meta backends, it checks if any of the capabilities in the map match the system capability.
68+
// For concrete backends, it delegates to SystemState.IsBackendCompatible.
69+
func (m *GalleryBackend) IsCompatibleWith(systemState *system.SystemState) bool {
70+
if systemState == nil {
71+
return true
72+
}
73+
74+
// Meta backends are compatible if the system capability matches one of the keys
75+
if m.IsMeta() {
76+
capability := systemState.Capability(m.CapabilitiesMap)
77+
_, exists := m.CapabilitiesMap[capability]
78+
return exists
79+
}
80+
81+
// For concrete backends, delegate to the system package
82+
return systemState.IsBackendCompatible(m.Name, m.URI)
83+
}
84+
6685
func (m *GalleryBackend) SetInstalled(installed bool) {
6786
m.Installed = installed
6887
}

‎core/gallery/backends_test.go‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,252 @@ var _ = Describe("Gallery Backends", func() {
172172
Expect(nilMetaBackend.IsMeta()).To(BeFalse())
173173
})
174174

175+
It("should check IsCompatibleWith correctly for meta backends", func() {
176+
metaBackend := &GalleryBackend{
177+
Metadata: Metadata{
178+
Name: "meta-backend",
179+
},
180+
CapabilitiesMap: map[string]string{
181+
"nvidia": "nvidia-backend",
182+
"amd": "amd-backend",
183+
"default": "default-backend",
184+
},
185+
}
186+
187+
// Test with nil state - should be compatible
188+
Expect(metaBackend.IsCompatibleWith(nil)).To(BeTrue())
189+
190+
// Test with NVIDIA system - should be compatible (has nvidia key)
191+
nvidiaState := &system.SystemState{GPUVendor: "nvidia", VRAM: 8 * 1024 * 1024 * 1024}
192+
Expect(metaBackend.IsCompatibleWith(nvidiaState)).To(BeTrue())
193+
194+
// Test with default (no GPU) - should be compatible (has default key)
195+
defaultState := &system.SystemState{}
196+
Expect(metaBackend.IsCompatibleWith(defaultState)).To(BeTrue())
197+
})
198+
199+
Describe("IsCompatibleWith for concrete backends", func() {
200+
Context("CPU backends", func() {
201+
It("should be compatible on all systems", func() {
202+
cpuBackend := &GalleryBackend{
203+
Metadata: Metadata{
204+
Name: "cpu-llama-cpp",
205+
},
206+
URI: "quay.io/go-skynet/local-ai-backends:latest-cpu-llama-cpp",
207+
}
208+
Expect(cpuBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
209+
Expect(cpuBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
210+
Expect(cpuBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
211+
})
212+
})
213+
214+
Context("Darwin/Metal backends", func() {
215+
When("running on darwin", func() {
216+
BeforeEach(func() {
217+
if runtime.GOOS != "darwin" {
218+
Skip("Skipping darwin-specific tests on non-darwin system")
219+
}
220+
})
221+
222+
It("should be compatible for MLX backend", func() {
223+
mlxBackend := &GalleryBackend{
224+
Metadata: Metadata{
225+
Name: "mlx",
226+
},
227+
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx",
228+
}
229+
Expect(mlxBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
230+
})
231+
232+
It("should be compatible for metal-llama-cpp backend", func() {
233+
metalBackend := &GalleryBackend{
234+
Metadata: Metadata{
235+
Name: "metal-llama-cpp",
236+
},
237+
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp",
238+
}
239+
Expect(metalBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
240+
})
241+
})
242+
243+
When("running on non-darwin", func() {
244+
BeforeEach(func() {
245+
if runtime.GOOS == "darwin" {
246+
Skip("Skipping non-darwin-specific tests on darwin system")
247+
}
248+
})
249+
250+
It("should NOT be compatible for MLX backend", func() {
251+
mlxBackend := &GalleryBackend{
252+
Metadata: Metadata{
253+
Name: "mlx",
254+
},
255+
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx",
256+
}
257+
Expect(mlxBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
258+
})
259+
260+
It("should NOT be compatible for metal-llama-cpp backend", func() {
261+
metalBackend := &GalleryBackend{
262+
Metadata: Metadata{
263+
Name: "metal-llama-cpp",
264+
},
265+
URI: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp",
266+
}
267+
Expect(metalBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
268+
})
269+
})
270+
})
271+
272+
Context("NVIDIA/CUDA backends", func() {
273+
When("running on non-darwin", func() {
274+
BeforeEach(func() {
275+
if runtime.GOOS == "darwin" {
276+
Skip("Skipping CUDA tests on darwin system")
277+
}
278+
})
279+
280+
It("should NOT be compatible without nvidia GPU", func() {
281+
cudaBackend := &GalleryBackend{
282+
Metadata: Metadata{
283+
Name: "cuda12-llama-cpp",
284+
},
285+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp",
286+
}
287+
Expect(cudaBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
288+
Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse())
289+
})
290+
291+
It("should be compatible with nvidia GPU", func() {
292+
cudaBackend := &GalleryBackend{
293+
Metadata: Metadata{
294+
Name: "cuda12-llama-cpp",
295+
},
296+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-llama-cpp",
297+
}
298+
Expect(cudaBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
299+
})
300+
301+
It("should be compatible with cuda13 backend on nvidia GPU", func() {
302+
cuda13Backend := &GalleryBackend{
303+
Metadata: Metadata{
304+
Name: "cuda13-llama-cpp",
305+
},
306+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-llama-cpp",
307+
}
308+
Expect(cuda13Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
309+
})
310+
})
311+
})
312+
313+
Context("AMD/ROCm backends", func() {
314+
When("running on non-darwin", func() {
315+
BeforeEach(func() {
316+
if runtime.GOOS == "darwin" {
317+
Skip("Skipping AMD/ROCm tests on darwin system")
318+
}
319+
})
320+
321+
It("should NOT be compatible without AMD GPU", func() {
322+
rocmBackend := &GalleryBackend{
323+
Metadata: Metadata{
324+
Name: "rocm-llama-cpp",
325+
},
326+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp",
327+
}
328+
Expect(rocmBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
329+
Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse())
330+
})
331+
332+
It("should be compatible with AMD GPU", func() {
333+
rocmBackend := &GalleryBackend{
334+
Metadata: Metadata{
335+
Name: "rocm-llama-cpp",
336+
},
337+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-llama-cpp",
338+
}
339+
Expect(rocmBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
340+
})
341+
342+
It("should be compatible with hipblas backend on AMD GPU", func() {
343+
hipBackend := &GalleryBackend{
344+
Metadata: Metadata{
345+
Name: "hip-llama-cpp",
346+
},
347+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-hip-llama-cpp",
348+
}
349+
Expect(hipBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.AMD, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
350+
})
351+
})
352+
})
353+
354+
Context("Intel/SYCL backends", func() {
355+
When("running on non-darwin", func() {
356+
BeforeEach(func() {
357+
if runtime.GOOS == "darwin" {
358+
Skip("Skipping Intel/SYCL tests on darwin system")
359+
}
360+
})
361+
362+
It("should NOT be compatible without Intel GPU", func() {
363+
intelBackend := &GalleryBackend{
364+
Metadata: Metadata{
365+
Name: "intel-sycl-f16-llama-cpp",
366+
},
367+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp",
368+
}
369+
Expect(intelBackend.IsCompatibleWith(&system.SystemState{})).To(BeFalse())
370+
Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Nvidia, VRAM: 8 * 1024 * 1024 * 1024})).To(BeFalse())
371+
})
372+
373+
It("should be compatible with Intel GPU", func() {
374+
intelBackend := &GalleryBackend{
375+
Metadata: Metadata{
376+
Name: "intel-sycl-f16-llama-cpp",
377+
},
378+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-llama-cpp",
379+
}
380+
Expect(intelBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
381+
})
382+
383+
It("should be compatible with intel-sycl-f32 backend on Intel GPU", func() {
384+
intelF32Backend := &GalleryBackend{
385+
Metadata: Metadata{
386+
Name: "intel-sycl-f32-llama-cpp",
387+
},
388+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-llama-cpp",
389+
}
390+
Expect(intelF32Backend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
391+
})
392+
393+
It("should be compatible with intel-transformers backend on Intel GPU", func() {
394+
intelTransformersBackend := &GalleryBackend{
395+
Metadata: Metadata{
396+
Name: "intel-transformers",
397+
},
398+
URI: "quay.io/go-skynet/local-ai-backends:latest-intel-transformers",
399+
}
400+
Expect(intelTransformersBackend.IsCompatibleWith(&system.SystemState{GPUVendor: system.Intel, VRAM: 8 * 1024 * 1024 * 1024})).To(BeTrue())
401+
})
402+
})
403+
})
404+
405+
Context("Vulkan backends", func() {
406+
It("should be compatible on CPU-only systems", func() {
407+
// Vulkan backends don't have a specific GPU vendor requirement in the current logic
408+
// They are compatible if no other GPU-specific pattern matches
409+
vulkanBackend := &GalleryBackend{
410+
Metadata: Metadata{
411+
Name: "vulkan-llama-cpp",
412+
},
413+
URI: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-llama-cpp",
414+
}
415+
// Vulkan doesn't have vendor-specific filtering in current implementation
416+
Expect(vulkanBackend.IsCompatibleWith(&system.SystemState{})).To(BeTrue())
417+
})
418+
})
419+
})
420+
175421
It("should find best backend from meta based on system capabilities", func() {
176422

177423
metaBackend := &GalleryBackend{

‎core/gallery/gallery.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,16 @@ func AvailableGalleryModels(galleries []config.Gallery, systemState *system.Syst
226226

227227
// List available backends
228228
func AvailableBackends(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
229+
return availableBackendsWithFilter(galleries, systemState, true)
230+
}
231+
232+
// AvailableBackendsUnfiltered returns all available backends without filtering by system capability.
233+
func AvailableBackendsUnfiltered(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
234+
return availableBackendsWithFilter(galleries, systemState, false)
235+
}
236+
237+
// availableBackendsWithFilter is a helper function that lists available backends with optional filtering.
238+
func availableBackendsWithFilter(galleries []config.Gallery, systemState *system.SystemState, filterByCapability bool) (GalleryElements[*GalleryBackend], error) {
229239
var backends []*GalleryBackend
230240

231241
systemBackends, err := ListSystemBackends(systemState)
@@ -241,7 +251,17 @@ func AvailableBackends(galleries []config.Gallery, systemState *system.SystemSta
241251
if err != nil {
242252
return nil, err
243253
}
244-
backends = append(backends, galleryBackends...)
254+
255+
// Filter backends by system capability if requested
256+
if filterByCapability {
257+
for _, backend := range galleryBackends {
258+
if backend.IsCompatibleWith(systemState) {
259+
backends = append(backends, backend)
260+
}
261+
}
262+
} else {
263+
backends = append(backends, galleryBackends...)
264+
}
245265
}
246266

247267
return backends, nil

‎core/http/routes/ui_api.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,12 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
617617
installedBackendsCount = len(installedBackends)
618618
}
619619

620+
// Get the detected system capability
621+
detectedCapability := ""
622+
if appConfig.SystemState != nil {
623+
detectedCapability = appConfig.SystemState.DetectedCapability()
624+
}
625+
620626
return c.JSON(200, map[string]interface{}{
621627
"backends": backendsJSON,
622628
"repositories": appConfig.BackendGalleries,
@@ -629,6 +635,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
629635
"totalPages": totalPages,
630636
"prevPage": prevPage,
631637
"nextPage": nextPage,
638+
"systemCapability": detectedCapability,
632639
})
633640
})
634641

‎core/http/views/backends.html‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ <h1 class="hero-title">
5454
<span class="font-semibold text-cyan-300" x-text="installedBackends"></span>
5555
<span class="text-[#94A3B8] ml-1">installed</span>
5656
</a>
57+
<div class="flex items-center bg-[#101827] rounded-lg px-4 py-2 border border-[#38BDF8]/30">
58+
<i class="fas fa-microchip text-[#38BDF8] mr-2"></i>
59+
<span class="text-[#94A3B8] mr-1">Capability:</span>
60+
<span class="font-semibold text-[#38BDF8]" x-text="systemCapability"></span>
61+
</div>
5762
<a href="https://localai.io/backends/" target="_blank" class="btn-primary">
5863
<i class="fas fa-info-circle mr-2"></i>
5964
<span>Documentation</span>
@@ -588,6 +593,7 @@ <h3 class="text-xl font-semibold text-gray-900 dark:text-white" x-text="selected
588593
totalPages: 1,
589594
availableBackends: 0,
590595
installedBackends: 0,
596+
systemCapability: '',
591597
selectedBackend: null,
592598
jobProgress: {},
593599
notifications: [],
@@ -683,6 +689,7 @@ <h3 class="text-xl font-semibold text-gray-900 dark:text-white" x-text="selected
683689
this.totalPages = data.totalPages || 1;
684690
this.availableBackends = data.availableBackends || 0;
685691
this.installedBackends = data.installedBackends || 0;
692+
this.systemCapability = data.systemCapability || 'default';
686693
} catch (error) {
687694
console.error('Error fetching backends:', error);
688695
} finally {

0 commit comments

Comments
 (0)