Implement enhanced error handling and path resolution for Llama engine scanning

- Introduced a new function to generate human-readable error messages for subprocess failures during help command execution.
- Updated the `scan_llama_engine_version` function to utilize the new path resolution logic for executable binaries, improving robustness.
- Added a new module for resolving Llama server executable paths and their working directories, ensuring compatibility with different installation layouts.
- Enhanced tests to cover new error handling scenarios and path resolution logic, ensuring comprehensive validation of the scanning process.

Author: lapy

backend/engine_param_scanner.py

@@ -28,6 +28,41 @@ def _clear_llama_flags_cache() -> None:
 HELP_TIMEOUT = 90
 
 
+def _help_subprocess_failure_message(
+    returncode: int,
+    argv0: str,
+    *,
+    empty_stdout: bool,
+    scan_engine: Optional[str] = None,
+) -> str:
+    """Human-readable scan failure (126/127 often mean exec/loader/shebang issues)."""
+    exe = argv0 or "(unknown)"
+    tail = " (no stdout from --help; output may be missing or only on stderr)" if empty_stdout else ""
+    if returncode == 127:
+        head = (
+            f"process exited with code 127{tail}: the program could not be run (POSIX 127 — often "
+            f"“not found” at exec or in a wrapper). Executable: {exe}. "
+        )
+        if scan_engine == "lmdeploy":
+            return head + (
+                "For LMDeploy: `lmdeploy` is usually a script; fix the venv shebang Python or a stale `venv_path`."
+            )
+        if scan_engine in ("llama_cpp", "ik_llama"):
+            return head + (
+                "For llama.cpp / ik_llama: wrong arch or libc (e.g. glibc binary on musl), missing shared "
+                "libraries (CUDA/GGML — `.so` search path), bad `binary_path`, or a wrapper with a broken "
+                "shebang. Run `file` on the binary and the same `--help` in the API container; ensure "
+                "`LD_LIBRARY_PATH` includes the directory with ggml/llama shared libs (often `build/bin` next to the build)."
+            )
+        return head + "Check shebang, dynamic linker, and PATH/LD_LIBRARY_PATH."
+    if returncode == 126:
+        return (
+            f"process exited with code 126{tail}: cannot execute (permission denied or not a valid executable). "
+            f"Executable: {exe}"
+        )
+    return f"process exited with code {returncode}{tail}"
+
+
 def _abs_path(p: str) -> str:
     if not p:
         return p
@@ -41,6 +76,7 @@ def _run_help_argv(
     *,
     cwd: Optional[str] = None,
     extra_env: Optional[dict] = None,
+    scan_engine: Optional[str] = None,
 ) -> Tuple[str, Optional[str]]:
     env = os.environ.copy()
     if extra_env:
@@ -56,11 +92,18 @@ def _run_help_argv(
             env=env,
         )
         text = r.stdout or ""
+        argv0 = argv[0] if argv else ""
         if not text.strip():
+            if r.returncode != 0:
+                return "", _help_subprocess_failure_message(
+                    r.returncode, argv0, empty_stdout=True, scan_engine=scan_engine
+                )
             return "", "empty help output"
         if r.returncode != 0:
             # Caller may still parse stdout when --help printed despite non-zero exit.
-            return text, f"process exited with code {r.returncode}"
+            return text, _help_subprocess_failure_message(
+                r.returncode, argv0, empty_stdout=False, scan_engine=scan_engine
+            )
         return text, None
     except subprocess.TimeoutExpired:
         return "", "timeout"
@@ -72,44 +115,45 @@ def _run_help_argv(
 
 def scan_llama_engine_version(engine: str, version_row: dict) -> dict:
     """engine: llama_cpp | ik_llama"""
+    from backend.llama_server_exec import (
+        llama_help_ld_library_path,
+        resolve_llama_server_invocation_paths,
+    )
+
     binary_path = version_row.get("binary_path")
     if not binary_path:
         return _error_entry("", "missing binary_path")
     path = _abs_path(binary_path)
-    if not os.path.isfile(path):
-        return _error_entry(path, f"binary not found: {path}")
+    exec_path, work_cwd = resolve_llama_server_invocation_paths(path)
+    if not os.path.isfile(exec_path):
+        return _error_entry(exec_path, f"binary not found: {exec_path}")
 
-    binary_dir = os.path.dirname(path)
-    working_dir = binary_dir
-    if "/bin/" in binary_dir and "/build/bin/" not in binary_dir:
-        working_dir = binary_dir.replace("/bin/", "/build/bin/")
-    env_ld = binary_dir
-    if "/bin/" in env_ld and "/build/bin/" not in env_ld:
-        env_ld = env_ld.replace("/bin/", "/build/bin/")
+    ld_path = llama_help_ld_library_path(work_cwd)
 
     text, run_err = _run_help_argv(
-        [path, "--help"],
-        cwd=working_dir if os.path.isdir(working_dir) else None,
-        extra_env={"LD_LIBRARY_PATH": env_ld},
+        [exec_path, "--help"],
+        cwd=work_cwd if os.path.isdir(work_cwd) else None,
+        extra_env={"LD_LIBRARY_PATH": ld_path},
+        scan_engine=engine,
     )
     if not text.strip():
-        return _error_entry(path, run_err or "empty help output")
+        return _error_entry(exec_path, run_err or "empty help output")
     try:
         sections = parse_llama_help_to_sections(text, engine)
     except Exception as e:
         logger.exception("llama help parse failed")
-        return _error_entry(path, f"parse error: {e}")
+        return _error_entry(exec_path, f"parse error: {e}")
 
     n_params = sum(len(s.get("params") or []) for s in sections)
     if n_params == 0:
         msg = run_err or (
             "No CLI flags parsed from --help. If you only see GPU/CUDA lines, the binary may have exited "
             "before usage text was printed; try running it with --help in a shell."
         )
-        return _error_entry(path, msg)
+        return _error_entry(exec_path, msg)
 
     return {
-        "binary_path": path,
+        "binary_path": exec_path,
         "scanned_at": iso_now(),
         "scan_error": None,
         "sections": sections,
@@ -131,6 +175,7 @@ def scan_lmdeploy_version(version_row: dict) -> dict:
         [lmdeploy_bin, "serve", "api_server", "--help"],
         cwd=vdir,
         extra_env={"VIRTUAL_ENV": vdir, "PATH": f"{os.path.join(vdir, 'bin')}:{os.environ.get('PATH', '')}"},
+        scan_engine="lmdeploy",
     )
     if not text.strip():
         return _error_entry(lmdeploy_bin, run_err or "empty help output")
@@ -171,7 +216,25 @@ def scan_engine_version(store: Any, engine: str, version_row: dict) -> dict:
         return entry
 
     if engine in ("llama_cpp", "ik_llama"):
-        entry = scan_llama_engine_version(engine, version_row)
+        row = dict(version_row)
+        active = store.get_active_engine_version(engine)
+        if (
+            active
+            and active.get("version") == ver
+            and active.get("binary_path")
+        ):
+            try:
+                from backend.llama_engine_resolve import (
+                    get_active_llama_swap_binary_path,
+                    infer_llama_engine_for_binary,
+                )
+
+                swap_bin = get_active_llama_swap_binary_path(store)
+                if swap_bin and infer_llama_engine_for_binary(store, swap_bin) == engine:
+                    row["binary_path"] = swap_bin
+            except Exception as e:
+                logger.debug("Active llama-swap binary override skipped: %s", e)
+        entry = scan_llama_engine_version(engine, row)
     elif engine == "lmdeploy":
         entry = scan_lmdeploy_version(version_row)
     else:

backend/llama_engine_resolve.py

@@ -0,0 +1,57 @@
+"""Resolve active llama.cpp / ik_llama binary paths from the engines store (lightweight; no hf/swap imports)."""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+from backend.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+def abs_llama_binary_path(p: Optional[str]) -> str:
+    if not p:
+        return ""
+    if os.path.isabs(p):
+        return p
+    return os.path.join("/app", p.lstrip("/"))
+
+
+def get_active_llama_swap_binary_path(store: Any) -> Optional[str]:
+    """
+    Same resolution as llama-swap: first existing ``binary_path`` on active ``llama_cpp``,
+    else on active ``ik_llama``.
+    """
+    try:
+        for engine in ("llama_cpp", "ik_llama"):
+            active_version = store.get_active_engine_version(engine)
+            if not active_version or not active_version.get("binary_path"):
+                continue
+            binary_path = active_version["binary_path"]
+            if not os.path.isabs(binary_path):
+                binary_path = os.path.join("/app", binary_path)
+            if os.path.exists(binary_path):
+                return binary_path
+            abs_path = os.path.abspath(binary_path)
+            if os.path.exists(abs_path):
+                return abs_path
+        logger.warning("No active llama-cpp version found in data store")
+        return None
+    except Exception as e:
+        logger.error("Error getting active llama swap binary path: %s", e)
+        return None
+
+
+def infer_llama_engine_for_binary(store: Any, binary_path: str) -> str:
+    """Return ``llama_cpp`` or ``ik_llama`` depending on which active row references this path."""
+    try:
+        norm = os.path.abspath(abs_llama_binary_path(binary_path))
+        for eng in ("ik_llama", "llama_cpp"):
+            av = store.get_active_engine_version(eng)
+            if av and av.get("binary_path"):
+                if os.path.abspath(abs_llama_binary_path(av["binary_path"])) == norm:
+                    return eng
+    except Exception as e:
+        logger.debug("infer_llama_engine_for_binary: %s", e)
+    return "llama_cpp"

backend/llama_server_exec.py

@@ -0,0 +1,83 @@
+"""Resolve llama-server executable + cwd to match llama-swap ``cd …/build/bin && ./llama-server`` layout."""
+
+from __future__ import annotations
+
+import os
+from typing import Optional, Tuple
+
+
+def sibling_build_bin_from_install_bin(install_bin_dir: str) -> Optional[str]:
+    """
+    If ``install_bin_dir`` is ``PREFIX/bin`` (not already ``…/build/bin``), return ``PREFIX/build/bin``.
+
+    Handles paths that end with ``/bin`` but do not contain the substring ``/bin/`` (e.g. ``…/install/bin``).
+    """
+    wd = os.path.abspath(install_bin_dir)
+    norm = wd.rstrip(os.sep)
+    if os.path.basename(norm) != "bin":
+        return None
+    parent = os.path.dirname(norm)
+    if os.path.basename(parent) == "build":
+        return None
+    return os.path.join(parent, "build", "bin")
+
+
+def resolve_llama_server_invocation_paths(abs_binary_path: str) -> Tuple[str, str]:
+    """
+    Match ``generate_llama_swap_config`` launcher: ``cd working_dir && ./<binary>`` where
+    ``working_dir`` is ``…/build/bin`` when that directory exists and the stored path was under ``…/bin``.
+
+    Args:
+        abs_binary_path: Absolute path from engines.yaml (after ``/app`` join if needed).
+
+    Returns:
+        ``(executable_path, cwd)`` for subprocess (argv0 and working directory).
+    """
+    p = os.path.abspath(abs_binary_path)
+    working_dir = os.path.dirname(p)
+    binary_name = os.path.basename(p)
+    alt_dir = sibling_build_bin_from_install_bin(working_dir)
+    if alt_dir:
+        alt_exec = os.path.join(alt_dir, binary_name)
+        if os.path.isfile(alt_exec):
+            return os.path.abspath(alt_exec), os.path.abspath(alt_dir)
+    # Legacy layout: path segment contains ``/bin/`` (e.g. ``…/something/bin/extra``)
+    if "/bin/" in working_dir and "/build/bin/" not in working_dir:
+        legacy_alt = working_dir.replace("/bin/", "/build/bin/")
+        alt_exec = os.path.join(legacy_alt, binary_name)
+        if os.path.isfile(alt_exec):
+            return os.path.abspath(alt_exec), os.path.abspath(legacy_alt)
+    return p, os.path.abspath(working_dir)
+
+
+def llama_help_ld_library_path(binary_dir: str) -> str:
+    """Dirs to search for ggml/CUDA .so when running ``llama-server --help`` (scan / flag probes)."""
+    candidates: list[str] = []
+
+    def consider(path: str) -> None:
+        if not path:
+            return
+        ap = os.path.abspath(path)
+        if os.path.isdir(ap) and ap not in candidates:
+            candidates.append(ap)
+
+    consider(binary_dir)
+    sbb = sibling_build_bin_from_install_bin(binary_dir)
+    if sbb:
+        consider(sbb)
+    if "/bin/" in binary_dir and "/build/bin/" not in binary_dir:
+        consider(binary_dir.replace("/bin/", "/build/bin/"))
+    consider(os.path.join(binary_dir, "build", "bin"))
+    consider(os.path.join(binary_dir, "build"))
+    consider(os.path.join(binary_dir, "..", "build", "bin"))
+
+    seen = set(candidates)
+    tail = os.environ.get("LD_LIBRARY_PATH", "").strip()
+    if tail:
+        for part in tail.split(os.pathsep):
+            p = part.strip()
+            if p and p not in seen:
+                ap = os.path.abspath(p)
+                seen.add(ap)
+                candidates.append(ap)
+    return os.pathsep.join(candidates)