fix(llama-cpp): detect image input from /models architecture and /props

src/tau_coding/extensions/builtins/llama_cpp/router.py

@@ -19,6 +19,13 @@
 LLAMA_CPP_ROUTER_MIN_BUILD = 9688
 LLAMA_CPP_ROUTER_MAX_BUILD = 10595
 
+InputModalities = tuple[Literal["text", "image"], ...]
+
+# Every input modality llama.cpp can report. A report with any other value is
+# not trusted. Tau sends only text and images, so it ignores audio and video.
+_KNOWN_INPUT_MODALITIES = frozenset({"text", "image", "audio", "video"})
+_SENT_INPUT_MODALITIES = ("text", "image")
+
 RouterState = Literal[
     "loaded", "sleeping", "unloaded", "loading", "downloading", "failed", "unknown"
 ]
@@ -44,7 +51,7 @@ class RouterModel:
     id: str
     state: RouterState
     display_name: str | None = None
-    input_modalities: tuple[Literal["text", "image"], ...] | None = None
+    input_modalities: InputModalities | None = None
     failed: bool = False
     downloaded_bytes: int | None = None
     download_total_bytes: int | None = None
@@ -125,19 +132,7 @@ async def list_router_models(
             state = cast(RouterState, value)
         else:
             state = "unknown"
-        architecture = raw.get("architecture")
-        modalities_raw = (
-            architecture.get("input_modalities") if isinstance(architecture, Mapping) else None
-        )
-        modalities = None
-        if (
-            isinstance(modalities_raw, list)
-            and modalities_raw
-            and all(item in {"text", "image"} for item in modalities_raw)
-        ):
-            modalities = cast(
-                tuple[Literal["text", "image"], ...], tuple(dict.fromkeys(modalities_raw))
-            )
+        modalities = reported_input_modalities(raw)
         display = raw.get("name", raw.get("display_name"))
         downloaded_bytes, download_total_bytes = _download_progress(status)
         result.append(
@@ -156,6 +151,34 @@ async def list_router_models(
     return tuple(result)
 
 
+def reported_input_modalities(model: Mapping[str, object]) -> InputModalities | None:
+    """Return the text and image inputs that a llama.cpp ``/models`` entry reports.
+
+    llama.cpp reports them as ``architecture.input_modalities``. Other servers may
+    use a top-level ``input_modalities`` or ``modalities`` list. Returns None when
+    the entry reports nothing, or reports a value Tau does not know.
+    """
+    architecture = model.get("architecture")
+    reported = architecture.get("input_modalities") if isinstance(architecture, Mapping) else None
+    if reported is None:
+        reported = model.get("input_modalities", model.get("modalities"))
+    if not isinstance(reported, list) or not reported:
+        return None
+    if not all(isinstance(entry, str) and entry in _KNOWN_INPUT_MODALITIES for entry in reported):
+        return None
+    sent = tuple(entry for entry in _SENT_INPUT_MODALITIES if entry in reported)
+    return cast(InputModalities, sent) or None
+
+
+def props_input_modalities(props: object) -> InputModalities | None:
+    """Return the inputs that ``modalities.vision`` in a standard server's ``/props`` reports."""
+    modalities = props.get("modalities") if isinstance(props, Mapping) else None
+    vision = modalities.get("vision") if isinstance(modalities, Mapping) else None
+    if not isinstance(vision, bool):
+        return None
+    return ("text", "image") if vision else ("text",)
+
+
 def _download_progress(status: object) -> tuple[int | None, int | None]:
     if not isinstance(status, Mapping):
         return None, None
@@ -311,5 +334,7 @@ def _server_error_detail(response: httpx.Response) -> str | None:
     "detect_router",
     "list_router_models",
     "mutate_router_model",
+    "props_input_modalities",
+    "reported_input_modalities",
     "watch_router_download_progress",
 ]

src/tau_coding/extensions/builtins/llama_cpp/service.py

@@ -6,7 +6,7 @@
 import secrets
 from collections.abc import Callable, Mapping
 from contextlib import suppress
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import UTC, datetime
 from types import MappingProxyType
 from typing import Literal, cast
@@ -70,6 +70,8 @@
     detect_router,
     list_router_models,
     mutate_router_model,
+    props_input_modalities,
+    reported_input_modalities,
     watch_router_download_progress,
 )
 from .state import (
@@ -322,6 +324,9 @@ async def discover(
             )
             payload = _json_object(response, "/v1/models")
             models = _parse_models(payload)
+            if len(models) == 1 and models[0].input_modalities is None:
+                # Older llama.cpp builds report vision only in /props.
+                models = (await self._inputs_from_props(client, headers, models[0]),)
             return LlamaCppDiscovery(
                 self.endpoint,
                 models,
@@ -1439,6 +1444,29 @@ async def _detect_router_safely(
                 ),
             )
 
+    async def _inputs_from_props(
+        self,
+        client: httpx.AsyncClient,
+        headers: Mapping[str, str],
+        model: ProviderModel,
+    ) -> ProviderModel:
+        """Return ``model`` with the inputs that ``/props`` reports, if it reports any.
+
+        llama.cpp builds before October 2026 report vision only in ``/props``.
+        On a standard server it describes the one loaded model. A failed or
+        unusable response leaves the model unchanged.
+        """
+        try:
+            response = await client.get(self.endpoint.server_root + "/props", headers=dict(headers))
+        except asyncio.CancelledError:
+            raise
+        except httpx.HTTPError:
+            return model
+        if response.status_code != 200:
+            return model
+        modalities = props_input_modalities(_safe_json(response))
+        return model if modalities is None else replace(model, input_modalities=modalities)
+
     async def _get(
         self,
         client: httpx.AsyncClient,
@@ -1562,7 +1590,7 @@ def _parse_models(payload: Mapping[str, object]) -> tuple[ProviderModel, ...]:
         ids.add(model_id)
         display = item.get("name", item.get("display_name"))
         display_name = display if isinstance(display, str) and display.strip() else model_id
-        modalities = _modalities(item)
+        modalities = reported_input_modalities(item)
         context_window = _reported_context_window(item)
         models.append(
             ProviderModel(
@@ -1578,16 +1606,6 @@ def _parse_models(payload: Mapping[str, object]) -> tuple[ProviderModel, ...]:
     return tuple(models)
 
 
-def _modalities(item: Mapping[str, object]) -> tuple[Literal["text", "image"], ...] | None:
-    value = item.get("input_modalities", item.get("modalities"))
-    if not isinstance(value, list) or not value:
-        return None
-    if not all(isinstance(entry, str) and entry in {"text", "image"} for entry in value):
-        return None
-    result = tuple(dict.fromkeys(value))
-    return cast(tuple[Literal["text", "image"], ...], result) or None
-
-
 def _reported_context_window(item: Mapping[str, object]) -> int | None:
     # These names are accepted only when directly reported by a server. Tau
     # never derives one from a model family, max tokens, or a 128K convention.

tests/test_llama_cpp_extension.py

@@ -19,6 +19,8 @@
 from tau_coding.extensions.builtins.llama_cpp.huggingface import discover_hf_token
 from tau_coding.extensions.builtins.llama_cpp.router import (
     RouterCapability,
+    props_input_modalities,
+    reported_input_modalities,
     watch_router_download_progress,
 )
 from tau_coding.extensions.builtins.llama_cpp.service import (
@@ -406,6 +408,135 @@ def handler(request: httpx.Request) -> httpx.Response:
     await client.aclose()
 
 
+@pytest.mark.parametrize(
+    ("model", "expected"),
+    [
+        ({"architecture": {"input_modalities": ["text", "image"]}}, ("text", "image")),
+        # llama.cpp also reports audio and video; Tau keeps only what it can send.
+        ({"architecture": {"input_modalities": ["text", "image", "audio"]}}, ("text", "image")),
+        ({"architecture": {"input_modalities": ["text", "audio", "video"]}}, ("text",)),
+        ({"input_modalities": ["text", "image"]}, ("text", "image")),
+        ({"modalities": ["text"]}, ("text",)),
+        ({"architecture": {"input_modalities": ["text", "unknown"]}}, None),
+        ({"architecture": {"input_modalities": ["audio"]}}, None),
+        ({"architecture": {"input_modalities": ["text", {"kind": "image"}]}}, None),
+        ({"architecture": {"input_modalities": []}}, None),
+        ({"architecture": "text"}, None),
+        ({}, None),
+    ],
+)
+def test_reported_input_modalities_follows_llama_cpp_architecture(
+    model: Mapping[str, object], expected: tuple[str, ...] | None
+) -> None:
+    assert reported_input_modalities(model) == expected
+
+
+@pytest.mark.anyio
+async def test_discovery_reads_llama_cpp_architecture_modalities(tmp_path: Path) -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/health":
+            return httpx.Response(200, json={"status": "ok"})
+        return httpx.Response(
+            200,
+            json={
+                "data": [
+                    {
+                        "id": "vision-local",
+                        "object": "model",
+                        "architecture": {
+                            "input_modalities": ["text", "image", "audio"],
+                            "output_modalities": ["text"],
+                        },
+                    }
+                ]
+            },
+        )
+
+    client, _ = _client(handler)
+    service, _, _ = _service(tmp_path, client=client)
+    discovery = await service.discover(ResolvedProviderAuth())
+    assert discovery.models[0].input_modalities == ("text", "image")
+    await client.aclose()
+
+
+@pytest.mark.parametrize(
+    ("props", "expected"),
+    [
+        ({"modalities": {"vision": True, "audio": False}}, ("text", "image")),
+        ({"modalities": {"vision": False, "audio": True}}, ("text",)),
+        ({"modalities": {"vision": "yes"}}, None),
+        ({"modalities": None}, None),
+        ({}, None),
+        (None, None),
+    ],
+)
+def test_props_input_modalities_reads_only_a_boolean_vision_flag(
+    props: object, expected: tuple[str, ...] | None
+) -> None:
+    assert props_input_modalities(props) == expected
+
+
+def _standard_server(
+    models: list[dict[str, object]], props: httpx.Response
+) -> Callable[[httpx.Request], httpx.Response]:
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/health":
+            return httpx.Response(200, json={"status": "ok"})
+        if request.url.path == "/props":
+            return props
+        return httpx.Response(200, json={"data": models})
+
+    return handler
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize(
+    ("models", "props", "expected"),
+    [
+        # Builds before llama.cpp #29987 report vision only in /props.
+        (
+            [{"id": "vision-local"}],
+            httpx.Response(200, json={"role": "model", "modalities": {"vision": True}}),
+            ("text", "image"),
+        ),
+        (
+            [{"id": "text-local"}],
+            httpx.Response(200, json={"modalities": {"vision": False}}),
+            ("text",),
+        ),
+        # A failing /props never fails discovery.
+        ([{"id": "vision-local"}], httpx.Response(500), None),
+        # /models wins over /props.
+        (
+            [{"id": "vision-local", "architecture": {"input_modalities": ["text"]}}],
+            httpx.Response(200, json={"modalities": {"vision": True}}),
+            ("text",),
+        ),
+    ],
+)
+async def test_standard_discovery_falls_back_to_props_vision(
+    tmp_path: Path,
+    models: list[dict[str, object]],
+    props: httpx.Response,
+    expected: tuple[str, ...] | None,
+) -> None:
+    client, _ = _client(_standard_server(models, props))
+    service, _, _ = _service(tmp_path, client=client)
+    discovery = await service.discover(ResolvedProviderAuth())
+    assert discovery.models[0].input_modalities == expected
+    await client.aclose()
+
+
+@pytest.mark.anyio
+async def test_props_vision_is_not_applied_to_several_models(tmp_path: Path) -> None:
+    props = httpx.Response(200, json={"modalities": {"vision": True}})
+    client, _ = _client(_standard_server([{"id": "first"}, {"id": "second"}], props))
+    service, _, _ = _service(tmp_path, client=client)
+    discovery = await service.discover(ResolvedProviderAuth())
+    assert [model.input_modalities for model in discovery.models] == [None, None]
+    await client.aclose()
+
+
 @pytest.mark.anyio
 async def test_dormant_cache_and_offline_refresh_are_network_free(tmp_path: Path) -> None:
     dormant, _, _ = _service(tmp_path / "dormant", environment={})

website/content/guides/local-inference.md

@@ -126,6 +126,15 @@ headless startup requires the exact `--model`; Tau never silently chooses a
 different explicit model. Local inference is not an automatic global-provider
 fallback.
 
+Image input follows what the server reports. A model whose `/v1/models` entry
+lists `image` in `architecture.input_modalities` (llama.cpp reports this when it
+was started with a vision projector, `--mmproj`) receives images, for example
+from the read tool. Reported `audio` and `video` inputs are ignored because Tau
+does not send them. For older single-model servers that report no usable inputs
+in `/v1/models`, Tau checks `/props.modalities.vision`. If neither endpoint
+reports usable capabilities, the model is treated as text-only; declare its
+image input with a [`catalog.toml`](/guides/providers-and-models/) entry instead.
+
 The safe integration snapshot is stored at
 `~/.tau/state/extensions/llama.cpp.json`. It contains only the normalized
 endpoint, selected model reference, allowlisted model metadata, and a timestamp.