Fix

Author: lapy

Dockerfile

@@ -1,7 +1,7 @@
 ################################################################################
 # llama.cpp Studio - Multi-stage Docker build
 ################################################################################
-ARG BASE_IMAGE=nvidia/cuda:12.9.1-devel-ubuntu22.04
+ARG BASE_IMAGE=ubuntu:22.04
 
 ################################################################################
 # Stage 1: Frontend Builder
@@ -31,7 +31,7 @@ FROM ${BASE_IMAGE} AS python-builder
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Install build dependencies in one layer
+# Install build dependencies required for python wheels and runtime compilation (llama.cpp)
 RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
     python3-venv \
@@ -50,6 +50,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     wget \
     ca-certificates \
+    pciutils \
+    lsb-release \
     && rm -rf /var/lib/apt/lists/* \
     && apt-get clean
 
@@ -80,10 +82,17 @@ ENV DEBIAN_FRONTEND=noninteractive \
     VENV_PATH=/opt/venv \
     PYTHONPATH=/app
 
-# Install runtime dependencies only
+# Install runtime dependencies (retain build toolchain for llama.cpp builds and GPU detection)
 RUN apt-get update && apt-get install -y --no-install-recommends \
     python3 \
     python3-venv \
+    python3-pip \
+    python3-dev \
+    build-essential \
+    cmake \
+    git \
+    pkg-config \
+    ninja-build \
     curl \
     wget \
     ca-certificates \
@@ -98,6 +107,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     mesa-vulkan-drivers \
     ocl-icd-libopencl1 \
     libnuma1 \
+    pciutils \
+    usbutils \
+    lshw \
     # Optional: ROCm (fails gracefully if unavailable)
     && (apt-get install -y --no-install-recommends rocminfo rocm-smi || echo "ROCm unavailable") \
     && rm -rf /var/lib/apt/lists/* \
@@ -123,6 +135,7 @@ WORKDIR /app
 COPY backend/ ./backend/
 COPY migrate_db.py ./
 COPY --from=frontend-builder /build/dist ./frontend/dist
+COPY frontend/public ./frontend/public
 
 # Create python symlink for compatibility
 RUN ln -sf /usr/bin/python3 /usr/bin/python

README.md

@@ -41,6 +41,7 @@ A professional AI model management platform for llama.cpp models and versions, d
 - **Real-time Updates**: WebSocket-based progress tracking and system monitoring
 - **Responsive Design**: Works on desktop and mobile devices
 - **System Status**: CPU, memory, disk, and GPU monitoring
+- **LMDeploy Installer**: Dedicated UI to install/remove LMDeploy at runtime with live logs
 - **Dark Mode**: Built-in theme support
 
 ## Quick Start
@@ -74,7 +75,7 @@ docker-compose -f docker-compose.rocm.yml up -d
 
 Prebuilt images are pushed to GitHub Container Registry whenever the `publish-docker` workflow runs.
 
-- `ghcr.io/<org-or-user>/llama-cpp-studio:latest` – standard image based on `nvidia/cuda:12.9.1-devel-ubuntu22.04`
+- `ghcr.io/<org-or-user>/llama-cpp-studio:latest` – standard image based on `ubuntu:22.04` with GPU tooling installed at runtime
 
 Pull the image from GHCR:
 
@@ -166,7 +167,7 @@ docker-compose up -d
 
 ### LMDeploy Requirement
 
-Safetensors execution relies on [LMDeploy](https://github.com/InternLM/lmdeploy). The official Docker image installs `lmdeploy` (pinned in `requirements.txt`, currently `0.10.2`) during build so the CLI is already available at runtime. If you are running outside the container, install it manually (`pip install lmdeploy`) or point `LMDEPLOY_BIN` to a custom binary. The runtime uses `lmdeploy serve turbomind` to expose an OpenAI-compatible server on port `2001`.
+Safetensors execution relies on [LMDeploy](https://github.com/InternLM/lmdeploy), but the base image intentionally omits it to keep Docker builds lightweight (critical for GitHub Actions). Use the **LMDeploy** page in the UI to install or remove LMDeploy inside the running container—installs happen via `pip` at runtime and logs are streamed live. The installer creates a dedicated virtual environment under `/app/data/lmdeploy/venv`, so the package lives on the writable volume and can be removed by deleting that folder. If you are running outside the container, you can still `pip install lmdeploy` manually or point `LMDEPLOY_BIN` to a custom binary. The runtime uses `lmdeploy serve turbomind` to expose an OpenAI-compatible server on port `2001`.
 
 ## Usage
 
@@ -260,7 +261,7 @@ Model names are shown in System Status after starting a model.
 - Run exactly one safetensors checkpoint at a time via LMDeploy
 - Configure tensor/pipeline parallelism, context length, temperature, and other runtime flags from the Model Library
 - Serves an OpenAI-compatible endpoint at `http://localhost:2001/v1/chat/completions`
-- CLI is preinstalled in the official Docker image; install it manually if running outside the container
+- Install LMDeploy on demand from the LMDeploy page (or manually via `pip`) before starting safetensors runtimes
 - Start/stop directly from the Safetensors panel; status is reported in System Status and the LMDeploy status chip
 
 ## Build Customization
@@ -282,7 +283,6 @@ Customize your build with:
 - **Custom CMake Flags**: Additional CMake configuration
 - **Compiler Flags**: CFLAGS and CXXFLAGS for optimization
 - **Git Patches**: Apply patches from GitHub PRs
-- **LMDeploy Version**: Override the `LMDEPLOY_VERSION` build arg to pin a specific CLI release inside the Docker image
 
 ### Example Build Configuration
 
@@ -350,6 +350,12 @@ The system automatically detects NVLink topology and applies appropriate strateg
 - `POST /api/models/safetensors/{model_id}/lmdeploy/stop` - Stop LMDeploy runtime
 - `GET /api/models/safetensors/lmdeploy/status` - LMDeploy manager status
 
+### LMDeploy Installer
+- `GET /api/lmdeploy/status` - Installer status (version, binary path, current operation)
+- `POST /api/lmdeploy/install` - Install LMDeploy via pip at runtime
+- `POST /api/lmdeploy/remove` - Remove LMDeploy from the runtime environment
+- `GET /api/lmdeploy/logs` - Tail the LMDeploy installer log
+
 ### llama.cpp Versions
 - `GET /api/llama-versions` - List installed versions
 - `GET /api/llama-versions/check-updates` - Check for updates

backend/main.py

@@ -10,7 +10,7 @@
 from contextlib import asynccontextmanager
 
 from backend.database import init_db, LlamaVersion
-from backend.routes import models, llama_versions, status, gpu_info, llama_version_manager
+from backend.routes import models, llama_versions, status, gpu_info, llama_version_manager, lmdeploy
 from backend.websocket_manager import websocket_manager
 from backend.huggingface import set_huggingface_token
 from backend.unified_monitor import unified_monitor
@@ -24,7 +24,7 @@
 def ensure_data_directories():
     """Ensure data directories exist and are writable"""
     data_dir = "/app/data"
-    subdirs = ["models", "configs", "logs", "llama-cpp"]
+    subdirs = ["models", "configs", "logs", "llama-cpp", "lmdeploy"]
 
     try:
         # Ensure main data directory exists
@@ -127,56 +127,56 @@ async def lifespan(app: FastAPI):
     global llama_swap_manager
 
     # Startup
-    # Ensure data directories exist and are writable
     ensure_data_directories()
-    
     await init_db()
-    
-    # Initialize configuration manager and update llama-swap config
-    
-    # Initialize Hugging Face API key from environment variable if available
+
     huggingface_api_key = os.getenv('HUGGINGFACE_API_KEY')
     if huggingface_api_key:
         set_huggingface_token(huggingface_api_key)
         logger.info("HuggingFace API key loaded from environment variable")
-    
-    # Initialize and start llama-swap
+
     from backend.llama_swap_manager import get_llama_swap_manager
     llama_swap_manager = get_llama_swap_manager()
-    
-    try:
-        await llama_swap_manager.start_proxy()
-        logger.info("llama-swap proxy started on port 2000")
-    except Exception as e:
-        logger.error(f"Failed to start llama-swap: {e}")
-        logger.warning("Multi-model serving unavailable")
-    
-    # Clean stale database state (since llama-swap was not running)
-    from backend.database import SessionLocal, RunningInstance, Model
+
+    from backend.database import SessionLocal, LlamaVersion, RunningInstance, Model
+    session = SessionLocal()
+    active_version = session.query(LlamaVersion).filter(LlamaVersion.is_active == True).first()
+    session.close()
+
+    if active_version and active_version.binary_path:
+        try:
+            await llama_swap_manager.start_proxy()
+            logger.info("llama-swap proxy started on port 2000")
+        except Exception as e:
+            logger.error(f"Failed to start llama-swap: {e}")
+            logger.warning("Multi-model serving unavailable")
+    else:
+        logger.warning(
+            "Skipping llama-swap start: no active llama.cpp version found. "
+            "Install or activate a llama.cpp build to enable multi-model serving."
+        )
+
     db = SessionLocal()
     try:
         stale_instances = db.query(RunningInstance).all()
         if stale_instances:
             logger.info(f"Cleaning {len(stale_instances)} stale instances")
             for instance in stale_instances:
-                # Update model status
                 model = db.query(Model).filter(Model.id == instance.model_id).first()
                 if model:
                     model.is_active = False
                 db.delete(instance)
             db.commit()
     finally:
         db.close()
-    
-    # Register all downloaded models with llama-swap
+
     try:
         await register_all_models_with_llama_swap()
     except Exception as e:
         logger.error(f"Failed to register models with llama-swap: {e}")
-    
-    # Start unified monitoring
+
     await unified_monitor.start_monitoring()
-    
+
     yield
 
     # Shutdown
@@ -226,6 +226,7 @@ async def lifespan(app: FastAPI):
 app.include_router(llama_version_manager.router, prefix="/api", tags=["llama-version-manager"])
 app.include_router(status.router, prefix="/api", tags=["status"])
 app.include_router(gpu_info.router, prefix="/api", tags=["gpu"])
+app.include_router(lmdeploy.router, prefix="/api", tags=["lmdeploy"])
 
 # Include monitoring routes
 from backend.routes import unified_monitoring

backend/routes/lmdeploy.py

@@ -0,0 +1,42 @@
+from typing import Dict, Optional
+
+from fastapi import APIRouter, HTTPException
+
+from backend.lmdeploy_installer import get_lmdeploy_installer
+
+router = APIRouter()
+
+
+@router.get("/lmdeploy/status")
+async def lmdeploy_installer_status() -> Dict:
+    installer = get_lmdeploy_installer()
+    return installer.status()
+
+
+@router.post("/lmdeploy/install")
+async def lmdeploy_install(request: Optional[Dict[str, str]] = None) -> Dict:
+    installer = get_lmdeploy_installer()
+    payload = request or {}
+    version = payload.get("version")
+    force_reinstall = bool(payload.get("force_reinstall"))
+    try:
+        return await installer.install(version=version, force_reinstall=force_reinstall)
+    except RuntimeError as exc:
+        raise HTTPException(status_code=409, detail=str(exc))
+
+
+@router.post("/lmdeploy/remove")
+async def lmdeploy_remove() -> Dict:
+    installer = get_lmdeploy_installer()
+    try:
+        return await installer.remove()
+    except RuntimeError as exc:
+        raise HTTPException(status_code=409, detail=str(exc))
+
+
+@router.get("/lmdeploy/logs")
+async def lmdeploy_logs(max_bytes: int = 8192) -> Dict[str, str]:
+    installer = get_lmdeploy_installer()
+    max_bytes = max(1024, min(max_bytes, 1024 * 1024))
+    return {"log": installer.read_log_tail(max_bytes)}
+

backend/routes/models.py

@@ -36,6 +36,7 @@
 from backend.llama_swap_config import get_supported_flags
 from backend.logging_config import get_logger
 from backend.lmdeploy_manager import get_lmdeploy_manager
+from backend.lmdeploy_installer import get_lmdeploy_installer
 import psutil
 
 router = APIRouter()
@@ -577,12 +578,14 @@ async def get_lmdeploy_config_endpoint(model_id: int, db: Session = Depends(get_
     max_context = manifest_entry.get("max_context_length") or metadata.get("max_context_length")
     config = (manifest_entry.get("lmdeploy") or {}).get("config") or get_default_lmdeploy_config(max_context)
     manager_status = get_lmdeploy_manager().status()
+    installer_status = get_lmdeploy_installer().status()
     return {
         "config": config,
         "metadata": metadata,
         "tensor_summary": tensor_summary,
         "max_context_length": max_context,
         "manager": manager_status,
+        "installer": installer_status,
     }
 
 
@@ -610,7 +613,21 @@ async def update_lmdeploy_config_endpoint(
 @router.get("/safetensors/lmdeploy/status")
 async def get_lmdeploy_status(db: Session = Depends(get_db)):
     """Return LMDeploy runtime status and running instance info."""
+    installer = get_lmdeploy_installer()
+    installer_status = installer.status()
+    if not installer_status.get("installed"):
+        raise HTTPException(
+            status_code=400,
+            detail="LMDeploy is not installed. Install it from the LMDeploy page before starting a runtime.",
+        )
+    if installer_status.get("operation"):
+        raise HTTPException(
+            status_code=409,
+            detail="An LMDeploy install/remove operation is still running. Try again once it finishes.",
+        )
+
     manager = get_lmdeploy_manager()
+    installer = get_lmdeploy_installer()
     running_instance = db.query(RunningInstance).filter(RunningInstance.runtime_type == "lmdeploy").first()
     instance_payload = None
     if running_instance:
@@ -621,6 +638,7 @@ async def get_lmdeploy_status(db: Session = Depends(get_db)):
         }
     return {
         "manager": manager.status(),
+        "installer": installer.status(),
         "running_instance": instance_payload
     }
 

backend/routes/status.py

@@ -5,6 +5,7 @@
 
 from backend.database import get_db, RunningInstance
 from backend.lmdeploy_manager import get_lmdeploy_manager
+from backend.lmdeploy_installer import get_lmdeploy_installer
 
 router = APIRouter()
 
@@ -42,6 +43,7 @@ async def get_system_status(db: Session = Depends(get_db)):
 
     lmdeploy_manager = get_lmdeploy_manager()
     lmdeploy_status = lmdeploy_manager.status()
+    installer_status = get_lmdeploy_installer().status()
 
     return {
         "system": {
@@ -69,6 +71,7 @@ async def get_system_status(db: Session = Depends(get_db)):
             "port": 2001,
             "endpoint": "http://localhost:2001/v1/chat/completions",
             "running": lmdeploy_status.get("running"),
-            "current_instance": lmdeploy_status.get("current_instance")
+            "current_instance": lmdeploy_status.get("current_instance"),
+            "installer": installer_status,
         }
     }

backend/tests/test_lmdeploy_installer.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+
+import pytest
+
+from backend.lmdeploy_installer import LMDeployInstaller
+
+
+@pytest.mark.asyncio
+async def test_install_prevents_parallel_operations(tmp_path: Path, monkeypatch):
+    installer = LMDeployInstaller(
+        log_path=str(tmp_path / "lmdeploy.log"),
+        state_path=str(tmp_path / "lmdeploy_state.json"),
+        base_dir=str(tmp_path / "lmdeploy"),
+    )
+
+    # Prevent the background task from executing pip in tests
+    def prevent_task(coro):
+        coro.close()
+
+    monkeypatch.setattr(installer, "_create_task", prevent_task)
+
+    result = await installer.install()
+    assert result["message"].startswith("LMDeploy installation started")
+
+    with pytest.raises(RuntimeError):
+        await installer.install()
+
+
+def test_status_reflects_detection(tmp_path: Path, monkeypatch):
+    installer = LMDeployInstaller(
+        log_path=str(tmp_path / "lmdeploy.log"),
+        state_path=str(tmp_path / "lmdeploy_state.json"),
+        base_dir=str(tmp_path / "lmdeploy"),
+    )
+
+    monkeypatch.setattr(installer, "_detect_installed_version", lambda: "0.10.0")
+    monkeypatch.setattr(installer, "_resolve_binary_path", lambda: "/opt/lmdeploy/bin/lmdeploy")
+
+    status = installer.status()
+    assert status["installed"] is True
+    assert status["version"] == "0.10.0"
+    assert status["binary_path"].endswith("lmdeploy")
+    assert status["venv_path"] == installer._venv_path
+