Refactor GPU detection and model metadata handling

- Changed asynchronous GPU detection functions to synchronous counterparts for improved performance and compatibility with blocking calls.
- Updated GPU information retrieval to utilize a caching mechanism, reducing redundant NVML calls and enhancing response times.
- Refactored model details fetching to support blocking operations, allowing for smoother integration with the frontend.
- Adjusted API routes to reflect changes in GPU detection and model metadata retrieval, ensuring consistent behavior across the application.

Author: lapy

backend/gpu_detector.py

@@ -6,6 +6,7 @@
 - CPU acceleration (OpenBLAS)
 """
 
+import asyncio
 import subprocess
 import json
 import os
@@ -149,13 +150,13 @@ def _query_nvml_cuda_version(initialized: bool = False) -> Optional[str]:
                 pass
 
 
-async def detect_nvidia_gpu() -> Optional[Dict]:
+def _detect_nvidia_gpu() -> Optional[Dict]:
     """Detect NVIDIA GPUs using pynvml and nvidia-smi"""
     cuda_version_hint: Optional[str] = None
 
     if pynvml is None:
         logger.debug("pynvml not available; attempting detection via nvidia-smi")
-        return await _detect_nvidia_via_smi()
+        return _detect_nvidia_via_smi()
 
     try:
         pynvml.nvmlInit()
@@ -166,7 +167,7 @@ async def detect_nvidia_gpu() -> Optional[Dict]:
             logger.debug(
                 "NVML reported zero NVIDIA devices; falling back to nvidia-smi detection"
             )
-            return await _detect_nvidia_via_smi(cuda_version_hint)
+            return _detect_nvidia_via_smi(cuda_version_hint)
 
         gpus = []
 
@@ -241,7 +242,7 @@ async def detect_nvidia_gpu() -> Optional[Dict]:
     except Exception as exc:
         logger.debug(f"Failed to detect NVIDIA GPUs via NVML: {exc}")
         # Fallback to nvidia-smi
-        return await _detect_nvidia_via_smi(cuda_version_hint)
+        return _detect_nvidia_via_smi(cuda_version_hint)
     finally:
         if pynvml is not None:
             try:
@@ -250,7 +251,7 @@ async def detect_nvidia_gpu() -> Optional[Dict]:
                 pass
 
 
-async def _detect_nvidia_via_smi(
+def _detect_nvidia_via_smi(
     cuda_version_hint: Optional[str] = None,
 ) -> Optional[Dict]:
     """Fallback NVIDIA detection using nvidia-smi"""
@@ -358,16 +359,16 @@ async def _detect_nvidia_via_smi(
 # ============================================================================
 
 
-async def detect_amd_gpu() -> Optional[Dict]:
+def _detect_amd_gpu() -> Optional[Dict]:
     """Detect AMD GPUs using rocm-smi or lspci"""
     try:
         # Try using rocm-smi first
-        amd_info = await _detect_amd_via_rocm()
+        amd_info = _detect_amd_via_rocm()
         if amd_info:
             return amd_info
 
         # Fallback to lspci
-        amd_info = await _detect_amd_via_lspci()
+        amd_info = _detect_amd_via_lspci()
         if amd_info:
             return amd_info
 
@@ -377,7 +378,7 @@ async def detect_amd_gpu() -> Optional[Dict]:
         return None
 
 
-async def _detect_amd_via_rocm() -> Optional[Dict]:
+def _detect_amd_via_rocm() -> Optional[Dict]:
     """Detect AMD GPUs using rocm-smi"""
     try:
         result = subprocess.run(
@@ -432,7 +433,7 @@ async def _detect_amd_via_rocm() -> Optional[Dict]:
         return None
 
 
-async def _detect_amd_via_lspci() -> Optional[Dict]:
+def _detect_amd_via_lspci() -> Optional[Dict]:
     """Detect AMD GPUs using lspci"""
     try:
         result = subprocess.run(
@@ -486,22 +487,19 @@ async def _detect_amd_via_lspci() -> Optional[Dict]:
 # ============================================================================
 
 
-async def get_gpu_info() -> Dict[str, any]:
-    """Get comprehensive GPU information (tries all vendors)"""
+def _collect_gpu_info() -> Dict[str, any]:
+    """Blocking GPU probe (NVML, nvidia-smi, rocm-smi, lspci). Run via asyncio.to_thread."""
     if _gpu_detection_disabled:
         return _cpu_only_response()
 
-    # Try NVIDIA first
-    nvidia_info = await detect_nvidia_gpu()
+    nvidia_info = _detect_nvidia_gpu()
     if nvidia_info:
         return nvidia_info
 
-    # Try AMD
-    amd_info = await detect_amd_gpu()
+    amd_info = _detect_amd_gpu()
     if amd_info:
         return amd_info
 
-    # No GPU detected
     return {
         "vendor": None,
         "cuda_version": "Unknown",
@@ -513,6 +511,13 @@ async def get_gpu_info() -> Dict[str, any]:
     }
 
 
+async def get_gpu_info() -> Dict[str, any]:
+    """Get comprehensive GPU information without blocking the event loop."""
+    if _gpu_detection_disabled:
+        return _cpu_only_response()
+    return await asyncio.to_thread(_collect_gpu_info)
+
+
 # ============================================================================
 # Backend Capability Detection
 # ============================================================================

backend/huggingface.py

@@ -1717,8 +1717,8 @@ def _get_attr_or_key(obj, key, default=None):
     return summary
 
 
-async def get_model_details(model_id: str) -> Dict:
-    """Get detailed model information including config and README"""
+def _get_model_details_blocking(model_id: str) -> Dict:
+    """Blocking Hugging Face API + config.json fetch (run via asyncio.to_thread)."""
     try:
         # Get model info with expanded data
         model_info = hf_api.model_info(model_id, expand=["cardData", "siblings"])
@@ -1773,6 +1773,11 @@ async def get_model_details(model_id: str) -> Dict:
         raise Exception(f"Failed to get model details: {e}")
 
 
+async def get_model_details(model_id: str) -> Dict:
+    """Get detailed model information including config and README."""
+    return await asyncio.to_thread(_get_model_details_blocking, model_id)
+
+
 async def download_model(
     huggingface_id: str, filename: str, model_format: str = "gguf"
 ) -> tuple[str, int]:

backend/routes/gpu_info.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter
 import multiprocessing
-from backend.gpu_detector import get_gpu_info as detect_gpu_info
+from backend.services.model_metadata import get_cached_gpu_info
 from backend.logging_config import get_logger
 
 router = APIRouter()
@@ -11,7 +11,7 @@
 async def get_gpu_info():
     """Detect GPU information and CPU capabilities via unified detector"""
     try:
-        info = await detect_gpu_info()
+        info = await get_cached_gpu_info()
     except Exception as exc:
         logger.exception("GPU detection failed: %s", exc)
         info = {

backend/routes/models.py

@@ -30,7 +30,6 @@
     search_models,
     set_huggingface_token,
     get_huggingface_token,
-    get_model_details,
     extract_quantization,
     list_grouped_safetensors_downloads,
     get_safetensors_manifest_entries,
@@ -213,6 +212,27 @@ class SafetensorsBundleRequest(BaseModel):
     files: List[Dict[str, Any]]
 
 
+_param_registry_cache: Dict[str, tuple] = {}
+_PARAM_REGISTRY_CACHE_TTL = 30.0
+
+
+def _param_registry_cache_key(store, engine: str) -> str:
+    active = store.get_active_engine_version(engine) if engine in (
+        "llama_cpp",
+        "ik_llama",
+        "lmdeploy",
+    ) else None
+    version = (active or {}).get("version") or ""
+    catalog_mtime = 0.0
+    try:
+        catalog_mtime = os.path.getmtime(
+            os.path.join(store._config_dir, "engine_params_catalog.yaml")
+        )
+    except OSError:
+        pass
+    return f"{engine}:{version}:{catalog_mtime}"
+
+
 @router.get("/param-registry")
 async def get_param_registry_endpoint(engine: str = "llama_cpp"):
     """Return param definitions from ``engine_params_catalog.yaml`` plus studio-only fields (read-only)."""
@@ -223,8 +243,16 @@ async def get_param_registry_endpoint(engine: str = "llama_cpp"):
     from backend.studio_engine_fields import studio_sections_for_engine
 
     store = get_store()
+    cache_key = _param_registry_cache_key(store, engine)
+    now = time.monotonic()
+    cached = _param_registry_cache.get(cache_key)
+    if cached and now - cached[1] < _PARAM_REGISTRY_CACHE_TTL:
+        return cached[0]
+
     if engine not in ("llama_cpp", "ik_llama", "lmdeploy"):
-        return registry_payload_from_entry(engine, None, [], has_active_engine=False)
+        payload = registry_payload_from_entry(engine, None, [], has_active_engine=False)
+        _param_registry_cache[cache_key] = (payload, now)
+        return payload
 
     studio = studio_sections_for_engine(engine)
     active = store.get_active_engine_version(engine)
@@ -239,9 +267,11 @@ async def get_param_registry_endpoint(engine: str = "llama_cpp"):
     if active and active.get("version"):
         entry = get_version_entry(store, engine, active["version"])
 
-    return registry_payload_from_entry(
+    payload = registry_payload_from_entry(
         engine, entry, studio, has_active_engine=has_active
     )
+    _param_registry_cache[cache_key] = (payload, now)
+    return payload
 
 
 @router.get("")
@@ -803,15 +833,16 @@ async def update_model_projector(
     }
 
 
-@router.get("/{model_id:path}/limits")
-async def get_model_limits(model_id: str):
+_limits_cache: Dict[str, tuple] = {}
+_LIMITS_CACHE_TTL = 300.0
+
+
+def _compute_model_limits(model: Dict[str, Any]) -> Dict[str, Optional[int]]:
     """
     Return model limits in an engine-agnostic way. For GGUF models with a manifest
     entry, uses the GGUF manifest; for safetensors, uses the safetensors manifest.
     Otherwise falls back to the Hugging Face model card (config.json / model info).
     """
-    store = get_store()
-    model = _get_model_or_404(store, model_id)
     hf_id = model.get("huggingface_id")
     if not hf_id:
         return {"max_context_length": None, "layer_count": None}
@@ -828,7 +859,9 @@ async def get_model_limits(model_id: str):
 
     if max_ctx is None or layer_count is None:
         try:
-            details = await get_model_details(hf_id)
+            from backend.huggingface import _get_model_details_blocking
+
+            details = _get_model_details_blocking(hf_id)
             config = details.get("config") or {}
             if max_ctx is None:
                 hf_max = details.get("model_max_length") or config.get(
@@ -840,15 +873,27 @@ async def get_model_limits(model_id: str):
                 for key in ("num_hidden_layers", "n_layer", "num_layers"):
                     val = config.get(key)
                     if isinstance(val, (int, float)) and val > 0:
-                        layer_count = (
-                            int(val) + 1
-                        )  # + output head for n_gpu_layers hint
+                        layer_count = int(val) + 1
                         break
         except Exception:
             pass
     return {"max_context_length": max_ctx, "layer_count": layer_count}
 
 
+@router.get("/{model_id:path}/limits")
+async def get_model_limits(model_id: str):
+    now = time.monotonic()
+    cached = _limits_cache.get(model_id)
+    if cached and now - cached[1] < _LIMITS_CACHE_TTL:
+        return cached[0]
+
+    store = get_store()
+    model = _get_model_or_404(store, model_id)
+    payload = await asyncio.to_thread(_compute_model_limits, model)
+    _limits_cache[model_id] = (payload, now)
+    return payload
+
+
 @router.get("/{model_id:path}/config")
 async def get_model_config(model_id: str):
     """Get model's llama.cpp configuration"""

backend/services/model_metadata.py

@@ -16,7 +16,7 @@
 
 # Lightweight cache for GPU info to avoid repeated NVML calls during rapid estimate requests
 _gpu_info_cache: Dict[str, Any] = {"data": None, "timestamp": 0.0}
-GPU_INFO_CACHE_TTL = 2.0  # seconds
+GPU_INFO_CACHE_TTL = 30.0  # seconds — shared by /api/gpu-info and VRAM estimates
 
 
 async def get_cached_gpu_info() -> Dict[str, Any]:

frontend/src/views/ModelConfig.test.js

@@ -125,6 +125,12 @@ function mountView() {
         LoadingState: { template: '<div>loading</div>' },
         EmptyState: { template: '<div><slot /></div>' },
         PageHeader: { template: '<div><slot name="start" /><slot name="title" /><slot name="actions" /></div>' },
+        Dialog: {
+          props: ['visible', 'header'],
+          emits: ['update:visible', 'show'],
+          template:
+            '<div v-if="visible" class="dialog-stub"><slot /><slot name="footer" /></div>',
+        },
       },
     },
   })
@@ -232,6 +238,12 @@ describe('ModelConfig', () => {
     await flushPromises()
 
     expect(wrapper.text()).toContain('legacy_temp')
+
+    await wrapper.get('button[data-label="Live preview"]').trigger('click')
+    await flushPromises()
+    await vi.runAllTimersAsync()
+    await flushPromises()
+
     expect(axios.post).toHaveBeenLastCalledWith(
       '/api/models/model-1/preview-llama-swap-cmd',
       {