diff --git a/src/tau_coding/extensions/builtins/llama_cpp/router.py b/src/tau_coding/extensions/builtins/llama_cpp/router.py index eea659d3f..ca286a952 100644 --- a/src/tau_coding/extensions/builtins/llama_cpp/router.py +++ b/src/tau_coding/extensions/builtins/llama_cpp/router.py @@ -19,6 +19,13 @@ LLAMA_CPP_ROUTER_MIN_BUILD = 9688 LLAMA_CPP_ROUTER_MAX_BUILD = 10595 +InputModalities = tuple[Literal["text", "image"], ...] + +# Every input modality llama.cpp can report. A report with any other value is +# not trusted. Tau sends only text and images, so it ignores audio and video. +_KNOWN_INPUT_MODALITIES = frozenset({"text", "image", "audio", "video"}) +_SENT_INPUT_MODALITIES = ("text", "image") + RouterState = Literal[ "loaded", "sleeping", "unloaded", "loading", "downloading", "failed", "unknown" ] @@ -44,7 +51,7 @@ class RouterModel: id: str state: RouterState display_name: str | None = None - input_modalities: tuple[Literal["text", "image"], ...] | None = None + input_modalities: InputModalities | None = None failed: bool = False downloaded_bytes: int | None = None download_total_bytes: int | None = None @@ -125,19 +132,7 @@ async def list_router_models( state = cast(RouterState, value) else: state = "unknown" - architecture = raw.get("architecture") - modalities_raw = ( - architecture.get("input_modalities") if isinstance(architecture, Mapping) else None - ) - modalities = None - if ( - isinstance(modalities_raw, list) - and modalities_raw - and all(item in {"text", "image"} for item in modalities_raw) - ): - modalities = cast( - tuple[Literal["text", "image"], ...], tuple(dict.fromkeys(modalities_raw)) - ) + modalities = reported_input_modalities(raw) display = raw.get("name", raw.get("display_name")) downloaded_bytes, download_total_bytes = _download_progress(status) result.append( @@ -156,6 +151,34 @@ async def list_router_models( return tuple(result) +def reported_input_modalities(model: Mapping[str, object]) -> InputModalities | None: + """Return the text and image inputs that a llama.cpp ``/models`` entry reports. + + llama.cpp reports them as ``architecture.input_modalities``. Other servers may + use a top-level ``input_modalities`` or ``modalities`` list. Returns None when + the entry reports nothing, or reports a value Tau does not know. + """ + architecture = model.get("architecture") + reported = architecture.get("input_modalities") if isinstance(architecture, Mapping) else None + if reported is None: + reported = model.get("input_modalities", model.get("modalities")) + if not isinstance(reported, list) or not reported: + return None + if not all(isinstance(entry, str) and entry in _KNOWN_INPUT_MODALITIES for entry in reported): + return None + sent = tuple(entry for entry in _SENT_INPUT_MODALITIES if entry in reported) + return cast(InputModalities, sent) or None + + +def props_input_modalities(props: object) -> InputModalities | None: + """Return the inputs that ``modalities.vision`` in a standard server's ``/props`` reports.""" + modalities = props.get("modalities") if isinstance(props, Mapping) else None + vision = modalities.get("vision") if isinstance(modalities, Mapping) else None + if not isinstance(vision, bool): + return None + return ("text", "image") if vision else ("text",) + + def _download_progress(status: object) -> tuple[int | None, int | None]: if not isinstance(status, Mapping): return None, None @@ -311,5 +334,7 @@ def _server_error_detail(response: httpx.Response) -> str | None: "detect_router", "list_router_models", "mutate_router_model", + "props_input_modalities", + "reported_input_modalities", "watch_router_download_progress", ] diff --git a/src/tau_coding/extensions/builtins/llama_cpp/service.py b/src/tau_coding/extensions/builtins/llama_cpp/service.py index 54d266955..1cf7f6c02 100644 --- a/src/tau_coding/extensions/builtins/llama_cpp/service.py +++ b/src/tau_coding/extensions/builtins/llama_cpp/service.py @@ -6,7 +6,7 @@ import secrets from collections.abc import Callable, Mapping from contextlib import suppress -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime from types import MappingProxyType from typing import Literal, cast @@ -70,6 +70,8 @@ detect_router, list_router_models, mutate_router_model, + props_input_modalities, + reported_input_modalities, watch_router_download_progress, ) from .state import ( @@ -322,6 +324,9 @@ async def discover( ) payload = _json_object(response, "/v1/models") models = _parse_models(payload) + if len(models) == 1 and models[0].input_modalities is None: + # Older llama.cpp builds report vision only in /props. + models = (await self._inputs_from_props(client, headers, models[0]),) return LlamaCppDiscovery( self.endpoint, models, @@ -1439,6 +1444,29 @@ async def _detect_router_safely( ), ) + async def _inputs_from_props( + self, + client: httpx.AsyncClient, + headers: Mapping[str, str], + model: ProviderModel, + ) -> ProviderModel: + """Return ``model`` with the inputs that ``/props`` reports, if it reports any. + + llama.cpp builds before October 2026 report vision only in ``/props``. + On a standard server it describes the one loaded model. A failed or + unusable response leaves the model unchanged. + """ + try: + response = await client.get(self.endpoint.server_root + "/props", headers=dict(headers)) + except asyncio.CancelledError: + raise + except httpx.HTTPError: + return model + if response.status_code != 200: + return model + modalities = props_input_modalities(_safe_json(response)) + return model if modalities is None else replace(model, input_modalities=modalities) + async def _get( self, client: httpx.AsyncClient, @@ -1562,7 +1590,7 @@ def _parse_models(payload: Mapping[str, object]) -> tuple[ProviderModel, ...]: ids.add(model_id) display = item.get("name", item.get("display_name")) display_name = display if isinstance(display, str) and display.strip() else model_id - modalities = _modalities(item) + modalities = reported_input_modalities(item) context_window = _reported_context_window(item) models.append( ProviderModel( @@ -1578,16 +1606,6 @@ def _parse_models(payload: Mapping[str, object]) -> tuple[ProviderModel, ...]: return tuple(models) -def _modalities(item: Mapping[str, object]) -> tuple[Literal["text", "image"], ...] | None: - value = item.get("input_modalities", item.get("modalities")) - if not isinstance(value, list) or not value: - return None - if not all(isinstance(entry, str) and entry in {"text", "image"} for entry in value): - return None - result = tuple(dict.fromkeys(value)) - return cast(tuple[Literal["text", "image"], ...], result) or None - - def _reported_context_window(item: Mapping[str, object]) -> int | None: # These names are accepted only when directly reported by a server. Tau # never derives one from a model family, max tokens, or a 128K convention. diff --git a/tests/test_llama_cpp_extension.py b/tests/test_llama_cpp_extension.py index 9ca7dc290..761c3f80a 100644 --- a/tests/test_llama_cpp_extension.py +++ b/tests/test_llama_cpp_extension.py @@ -19,6 +19,8 @@ from tau_coding.extensions.builtins.llama_cpp.huggingface import discover_hf_token from tau_coding.extensions.builtins.llama_cpp.router import ( RouterCapability, + props_input_modalities, + reported_input_modalities, watch_router_download_progress, ) from tau_coding.extensions.builtins.llama_cpp.service import ( @@ -406,6 +408,135 @@ def handler(request: httpx.Request) -> httpx.Response: await client.aclose() +@pytest.mark.parametrize( + ("model", "expected"), + [ + ({"architecture": {"input_modalities": ["text", "image"]}}, ("text", "image")), + # llama.cpp also reports audio and video; Tau keeps only what it can send. + ({"architecture": {"input_modalities": ["text", "image", "audio"]}}, ("text", "image")), + ({"architecture": {"input_modalities": ["text", "audio", "video"]}}, ("text",)), + ({"input_modalities": ["text", "image"]}, ("text", "image")), + ({"modalities": ["text"]}, ("text",)), + ({"architecture": {"input_modalities": ["text", "unknown"]}}, None), + ({"architecture": {"input_modalities": ["audio"]}}, None), + ({"architecture": {"input_modalities": ["text", {"kind": "image"}]}}, None), + ({"architecture": {"input_modalities": []}}, None), + ({"architecture": "text"}, None), + ({}, None), + ], +) +def test_reported_input_modalities_follows_llama_cpp_architecture( + model: Mapping[str, object], expected: tuple[str, ...] | None +) -> None: + assert reported_input_modalities(model) == expected + + +@pytest.mark.anyio +async def test_discovery_reads_llama_cpp_architecture_modalities(tmp_path: Path) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "vision-local", + "object": "model", + "architecture": { + "input_modalities": ["text", "image", "audio"], + "output_modalities": ["text"], + }, + } + ] + }, + ) + + client, _ = _client(handler) + service, _, _ = _service(tmp_path, client=client) + discovery = await service.discover(ResolvedProviderAuth()) + assert discovery.models[0].input_modalities == ("text", "image") + await client.aclose() + + +@pytest.mark.parametrize( + ("props", "expected"), + [ + ({"modalities": {"vision": True, "audio": False}}, ("text", "image")), + ({"modalities": {"vision": False, "audio": True}}, ("text",)), + ({"modalities": {"vision": "yes"}}, None), + ({"modalities": None}, None), + ({}, None), + (None, None), + ], +) +def test_props_input_modalities_reads_only_a_boolean_vision_flag( + props: object, expected: tuple[str, ...] | None +) -> None: + assert props_input_modalities(props) == expected + + +def _standard_server( + models: list[dict[str, object]], props: httpx.Response +) -> Callable[[httpx.Request], httpx.Response]: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + if request.url.path == "/props": + return props + return httpx.Response(200, json={"data": models}) + + return handler + + +@pytest.mark.anyio +@pytest.mark.parametrize( + ("models", "props", "expected"), + [ + # Builds before llama.cpp #29987 report vision only in /props. + ( + [{"id": "vision-local"}], + httpx.Response(200, json={"role": "model", "modalities": {"vision": True}}), + ("text", "image"), + ), + ( + [{"id": "text-local"}], + httpx.Response(200, json={"modalities": {"vision": False}}), + ("text",), + ), + # A failing /props never fails discovery. + ([{"id": "vision-local"}], httpx.Response(500), None), + # /models wins over /props. + ( + [{"id": "vision-local", "architecture": {"input_modalities": ["text"]}}], + httpx.Response(200, json={"modalities": {"vision": True}}), + ("text",), + ), + ], +) +async def test_standard_discovery_falls_back_to_props_vision( + tmp_path: Path, + models: list[dict[str, object]], + props: httpx.Response, + expected: tuple[str, ...] | None, +) -> None: + client, _ = _client(_standard_server(models, props)) + service, _, _ = _service(tmp_path, client=client) + discovery = await service.discover(ResolvedProviderAuth()) + assert discovery.models[0].input_modalities == expected + await client.aclose() + + +@pytest.mark.anyio +async def test_props_vision_is_not_applied_to_several_models(tmp_path: Path) -> None: + props = httpx.Response(200, json={"modalities": {"vision": True}}) + client, _ = _client(_standard_server([{"id": "first"}, {"id": "second"}], props)) + service, _, _ = _service(tmp_path, client=client) + discovery = await service.discover(ResolvedProviderAuth()) + assert [model.input_modalities for model in discovery.models] == [None, None] + await client.aclose() + + @pytest.mark.anyio async def test_dormant_cache_and_offline_refresh_are_network_free(tmp_path: Path) -> None: dormant, _, _ = _service(tmp_path / "dormant", environment={}) diff --git a/website/content/guides/local-inference.md b/website/content/guides/local-inference.md index 43a4cd7cb..b0d992b3b 100644 --- a/website/content/guides/local-inference.md +++ b/website/content/guides/local-inference.md @@ -126,6 +126,15 @@ headless startup requires the exact `--model`; Tau never silently chooses a different explicit model. Local inference is not an automatic global-provider fallback. +Image input follows what the server reports. A model whose `/v1/models` entry +lists `image` in `architecture.input_modalities` (llama.cpp reports this when it +was started with a vision projector, `--mmproj`) receives images, for example +from the read tool. Reported `audio` and `video` inputs are ignored because Tau +does not send them. For older single-model servers that report no usable inputs +in `/v1/models`, Tau checks `/props.modalities.vision`. If neither endpoint +reports usable capabilities, the model is treated as text-only; declare its +image input with a [`catalog.toml`](/guides/providers-and-models/) entry instead. + The safe integration snapshot is stored at `~/.tau/state/extensions/llama.cpp.json`. It contains only the normalized endpoint, selected model reference, allowlisted model metadata, and a timestamp.