From b723be83a9e9b5cedd4e74e22172094aae3f89d8 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:17:46 +0800 Subject: [PATCH 1/3] fix(llama-cpp): detect image input from /models architecture and /props llama.cpp reports a model's inputs as architecture.input_modalities, and builds before October 2026 only as modalities.vision in /props. Standard discovery read neither, and both parsers rejected lists that include audio or video, so vision models were treated as text-only. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../extensions/builtins/llama_cpp/router.py | 51 +++++-- .../extensions/builtins/llama_cpp/service.py | 41 ++++-- tests/test_llama_cpp_extension.py | 130 ++++++++++++++++++ website/content/guides/local-inference.md | 8 ++ 4 files changed, 205 insertions(+), 25 deletions(-) diff --git a/src/tau_coding/extensions/builtins/llama_cpp/router.py b/src/tau_coding/extensions/builtins/llama_cpp/router.py index eea659d3f..73d5db5c1 100644 --- a/src/tau_coding/extensions/builtins/llama_cpp/router.py +++ b/src/tau_coding/extensions/builtins/llama_cpp/router.py @@ -125,19 +125,7 @@ async def list_router_models( state = cast(RouterState, value) else: state = "unknown" - architecture = raw.get("architecture") - modalities_raw = ( - architecture.get("input_modalities") if isinstance(architecture, Mapping) else None - ) - modalities = None - if ( - isinstance(modalities_raw, list) - and modalities_raw - and all(item in {"text", "image"} for item in modalities_raw) - ): - modalities = cast( - tuple[Literal["text", "image"], ...], tuple(dict.fromkeys(modalities_raw)) - ) + modalities = reported_input_modalities(raw) display = raw.get("name", raw.get("display_name")) downloaded_bytes, download_total_bytes = _download_progress(status) result.append( @@ -156,6 +144,41 @@ async def list_router_models( return tuple(result) +# Input modalities llama.cpp can report. Tau sends only text and images, so the +# others are dropped; any other value makes the whole report untrusted. +_LLAMA_CPP_INPUT_MODALITIES = frozenset({"text", "image", "audio", "video"}) + + +def reported_input_modalities( + model: Mapping[str, object], +) -> tuple[Literal["text", "image"], ...] | None: + """Return the text and image inputs a llama.cpp ``/models`` entry reports. + + llama.cpp reports them under ``architecture.input_modalities``; a top-level + ``input_modalities`` or ``modalities`` list is accepted for other servers. + Returns None when nothing is reported or the report holds an unknown value. + """ + architecture = model.get("architecture") + value = architecture.get("input_modalities") if isinstance(architecture, Mapping) else None + if value is None: + value = model.get("input_modalities", model.get("modalities")) + if not isinstance(value, list) or not value: + return None + if not all(isinstance(entry, str) and entry in _LLAMA_CPP_INPUT_MODALITIES for entry in value): + return None + supported = tuple(dict.fromkeys(entry for entry in value if entry in {"text", "image"})) + return cast(tuple[Literal["text", "image"], ...], supported) or None + + +def props_input_modalities(props: object) -> tuple[Literal["text", "image"], ...] | None: + """Return the inputs a standard server's ``/props`` reports as ``modalities.vision``.""" + modalities = props.get("modalities") if isinstance(props, Mapping) else None + vision = modalities.get("vision") if isinstance(modalities, Mapping) else None + if not isinstance(vision, bool): + return None + return ("text", "image") if vision else ("text",) + + def _download_progress(status: object) -> tuple[int | None, int | None]: if not isinstance(status, Mapping): return None, None @@ -311,5 +334,7 @@ def _server_error_detail(response: httpx.Response) -> str | None: "detect_router", "list_router_models", "mutate_router_model", + "props_input_modalities", + "reported_input_modalities", "watch_router_download_progress", ] diff --git a/src/tau_coding/extensions/builtins/llama_cpp/service.py b/src/tau_coding/extensions/builtins/llama_cpp/service.py index 54d266955..60b270e82 100644 --- a/src/tau_coding/extensions/builtins/llama_cpp/service.py +++ b/src/tau_coding/extensions/builtins/llama_cpp/service.py @@ -6,7 +6,7 @@ import secrets from collections.abc import Callable, Mapping from contextlib import suppress -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime from types import MappingProxyType from typing import Literal, cast @@ -70,6 +70,8 @@ detect_router, list_router_models, mutate_router_model, + props_input_modalities, + reported_input_modalities, watch_router_download_progress, ) from .state import ( @@ -322,6 +324,8 @@ async def discover( ) payload = _json_object(response, "/v1/models") models = _parse_models(payload) + if len(models) == 1 and models[0].input_modalities is None: + models = (await self._with_props_modalities(client, headers, models[0]),) return LlamaCppDiscovery( self.endpoint, models, @@ -1439,6 +1443,29 @@ async def _detect_router_safely( ), ) + async def _with_props_modalities( + self, + client: httpx.AsyncClient, + headers: Mapping[str, str], + model: ProviderModel, + ) -> ProviderModel: + """Fill a single model's inputs from ``/props`` when ``/v1/models`` omits them. + + llama.cpp builds before October 2026 report image support only in + ``/props``, which describes the one loaded model of a standard server. + Any failure leaves the model unchanged. + """ + try: + response = await client.get(self.endpoint.server_root + "/props", headers=dict(headers)) + except asyncio.CancelledError: + raise + except httpx.HTTPError: + return model + if response.status_code != 200: + return model + modalities = props_input_modalities(_safe_json(response)) + return model if modalities is None else replace(model, input_modalities=modalities) + async def _get( self, client: httpx.AsyncClient, @@ -1562,7 +1589,7 @@ def _parse_models(payload: Mapping[str, object]) -> tuple[ProviderModel, ...]: ids.add(model_id) display = item.get("name", item.get("display_name")) display_name = display if isinstance(display, str) and display.strip() else model_id - modalities = _modalities(item) + modalities = reported_input_modalities(item) context_window = _reported_context_window(item) models.append( ProviderModel( @@ -1578,16 +1605,6 @@ def _parse_models(payload: Mapping[str, object]) -> tuple[ProviderModel, ...]: return tuple(models) -def _modalities(item: Mapping[str, object]) -> tuple[Literal["text", "image"], ...] | None: - value = item.get("input_modalities", item.get("modalities")) - if not isinstance(value, list) or not value: - return None - if not all(isinstance(entry, str) and entry in {"text", "image"} for entry in value): - return None - result = tuple(dict.fromkeys(value)) - return cast(tuple[Literal["text", "image"], ...], result) or None - - def _reported_context_window(item: Mapping[str, object]) -> int | None: # These names are accepted only when directly reported by a server. Tau # never derives one from a model family, max tokens, or a 128K convention. diff --git a/tests/test_llama_cpp_extension.py b/tests/test_llama_cpp_extension.py index 9ca7dc290..382cd704b 100644 --- a/tests/test_llama_cpp_extension.py +++ b/tests/test_llama_cpp_extension.py @@ -19,6 +19,8 @@ from tau_coding.extensions.builtins.llama_cpp.huggingface import discover_hf_token from tau_coding.extensions.builtins.llama_cpp.router import ( RouterCapability, + props_input_modalities, + reported_input_modalities, watch_router_download_progress, ) from tau_coding.extensions.builtins.llama_cpp.service import ( @@ -406,6 +408,134 @@ def handler(request: httpx.Request) -> httpx.Response: await client.aclose() +@pytest.mark.parametrize( + ("model", "expected"), + [ + ({"architecture": {"input_modalities": ["text", "image"]}}, ("text", "image")), + # llama.cpp also reports audio and video; Tau keeps only what it can send. + ({"architecture": {"input_modalities": ["text", "image", "audio"]}}, ("text", "image")), + ({"architecture": {"input_modalities": ["text", "audio", "video"]}}, ("text",)), + ({"input_modalities": ["text", "image"]}, ("text", "image")), + ({"modalities": ["text"]}, ("text",)), + ({"architecture": {"input_modalities": ["text", "unknown"]}}, None), + ({"architecture": {"input_modalities": ["audio"]}}, None), + ({"architecture": {"input_modalities": []}}, None), + ({"architecture": "text"}, None), + ({}, None), + ], +) +def test_reported_input_modalities_follows_llama_cpp_architecture( + model: Mapping[str, object], expected: tuple[str, ...] | None +) -> None: + assert reported_input_modalities(model) == expected + + +@pytest.mark.anyio +async def test_discovery_reads_llama_cpp_architecture_modalities(tmp_path: Path) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + return httpx.Response( + 200, + json={ + "data": [ + { + "id": "vision-local", + "object": "model", + "architecture": { + "input_modalities": ["text", "image", "audio"], + "output_modalities": ["text"], + }, + } + ] + }, + ) + + client, _ = _client(handler) + service, _, _ = _service(tmp_path, client=client) + discovery = await service.discover(ResolvedProviderAuth()) + assert discovery.models[0].input_modalities == ("text", "image") + await client.aclose() + + +@pytest.mark.parametrize( + ("props", "expected"), + [ + ({"modalities": {"vision": True, "audio": False}}, ("text", "image")), + ({"modalities": {"vision": False, "audio": True}}, ("text",)), + ({"modalities": {"vision": "yes"}}, None), + ({"modalities": None}, None), + ({}, None), + (None, None), + ], +) +def test_props_input_modalities_reads_only_a_boolean_vision_flag( + props: object, expected: tuple[str, ...] | None +) -> None: + assert props_input_modalities(props) == expected + + +def _standard_server( + models: list[dict[str, object]], props: httpx.Response +) -> Callable[[httpx.Request], httpx.Response]: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/health": + return httpx.Response(200, json={"status": "ok"}) + if request.url.path == "/props": + return props + return httpx.Response(200, json={"data": models}) + + return handler + + +@pytest.mark.anyio +@pytest.mark.parametrize( + ("models", "props", "expected"), + [ + # Builds before llama.cpp #29987 report vision only in /props. + ( + [{"id": "vision-local"}], + httpx.Response(200, json={"role": "model", "modalities": {"vision": True}}), + ("text", "image"), + ), + ( + [{"id": "text-local"}], + httpx.Response(200, json={"modalities": {"vision": False}}), + ("text",), + ), + # A failing /props never fails discovery. + ([{"id": "vision-local"}], httpx.Response(500), None), + # /models wins over /props. + ( + [{"id": "vision-local", "architecture": {"input_modalities": ["text"]}}], + httpx.Response(200, json={"modalities": {"vision": True}}), + ("text",), + ), + ], +) +async def test_standard_discovery_falls_back_to_props_vision( + tmp_path: Path, + models: list[dict[str, object]], + props: httpx.Response, + expected: tuple[str, ...] | None, +) -> None: + client, _ = _client(_standard_server(models, props)) + service, _, _ = _service(tmp_path, client=client) + discovery = await service.discover(ResolvedProviderAuth()) + assert discovery.models[0].input_modalities == expected + await client.aclose() + + +@pytest.mark.anyio +async def test_props_vision_is_not_applied_to_several_models(tmp_path: Path) -> None: + props = httpx.Response(200, json={"modalities": {"vision": True}}) + client, _ = _client(_standard_server([{"id": "first"}, {"id": "second"}], props)) + service, _, _ = _service(tmp_path, client=client) + discovery = await service.discover(ResolvedProviderAuth()) + assert [model.input_modalities for model in discovery.models] == [None, None] + await client.aclose() + + @pytest.mark.anyio async def test_dormant_cache_and_offline_refresh_are_network_free(tmp_path: Path) -> None: dormant, _, _ = _service(tmp_path / "dormant", environment={}) diff --git a/website/content/guides/local-inference.md b/website/content/guides/local-inference.md index 93219aed9..c4d579257 100644 --- a/website/content/guides/local-inference.md +++ b/website/content/guides/local-inference.md @@ -127,6 +127,14 @@ headless startup requires the exact `--model`; Tau never silently chooses a different explicit model. Local inference is not an automatic global-provider fallback. +Image input follows what the server reports. A model whose `/v1/models` entry +lists `image` in `architecture.input_modalities` (llama.cpp reports this when it +was started with a vision projector, `--mmproj`) receives images, for example +from the read tool. Reported `audio` and `video` inputs are ignored because Tau +does not send them. A server that reports no modalities, such as an older build +or a fork, is treated as text-only; declare its image input with a +[`catalog.toml`]({{< relref "./providers-and-models.md" >}}) entry instead. + The safe integration snapshot is stored at `~/.tau/state/extensions/llama.cpp.json`. It contains only the normalized endpoint, selected model reference, allowlisted model metadata, and a timestamp. From d150ff10da1f35da88b74236c11c73daaa2875a2 Mon Sep 17 00:00:00 2001 From: Onur Solmaz <2453968+osolmaz@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:27:10 +0800 Subject: [PATCH 2/3] refactor(llama-cpp): name the modality sets and type, constants at the top Co-Authored-By: Claude Opus 5.5 (1M context) --- .../extensions/builtins/llama_cpp/router.py | 44 +++++++++---------- .../extensions/builtins/llama_cpp/service.py | 13 +++--- tests/test_llama_cpp_extension.py | 1 + 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/tau_coding/extensions/builtins/llama_cpp/router.py b/src/tau_coding/extensions/builtins/llama_cpp/router.py index 73d5db5c1..ca286a952 100644 --- a/src/tau_coding/extensions/builtins/llama_cpp/router.py +++ b/src/tau_coding/extensions/builtins/llama_cpp/router.py @@ -19,6 +19,13 @@ LLAMA_CPP_ROUTER_MIN_BUILD = 9688 LLAMA_CPP_ROUTER_MAX_BUILD = 10595 +InputModalities = tuple[Literal["text", "image"], ...] + +# Every input modality llama.cpp can report. A report with any other value is +# not trusted. Tau sends only text and images, so it ignores audio and video. +_KNOWN_INPUT_MODALITIES = frozenset({"text", "image", "audio", "video"}) +_SENT_INPUT_MODALITIES = ("text", "image") + RouterState = Literal[ "loaded", "sleeping", "unloaded", "loading", "downloading", "failed", "unknown" ] @@ -44,7 +51,7 @@ class RouterModel: id: str state: RouterState display_name: str | None = None - input_modalities: tuple[Literal["text", "image"], ...] | None = None + input_modalities: InputModalities | None = None failed: bool = False downloaded_bytes: int | None = None download_total_bytes: int | None = None @@ -144,34 +151,27 @@ async def list_router_models( return tuple(result) -# Input modalities llama.cpp can report. Tau sends only text and images, so the -# others are dropped; any other value makes the whole report untrusted. -_LLAMA_CPP_INPUT_MODALITIES = frozenset({"text", "image", "audio", "video"}) - - -def reported_input_modalities( - model: Mapping[str, object], -) -> tuple[Literal["text", "image"], ...] | None: - """Return the text and image inputs a llama.cpp ``/models`` entry reports. +def reported_input_modalities(model: Mapping[str, object]) -> InputModalities | None: + """Return the text and image inputs that a llama.cpp ``/models`` entry reports. - llama.cpp reports them under ``architecture.input_modalities``; a top-level - ``input_modalities`` or ``modalities`` list is accepted for other servers. - Returns None when nothing is reported or the report holds an unknown value. + llama.cpp reports them as ``architecture.input_modalities``. Other servers may + use a top-level ``input_modalities`` or ``modalities`` list. Returns None when + the entry reports nothing, or reports a value Tau does not know. """ architecture = model.get("architecture") - value = architecture.get("input_modalities") if isinstance(architecture, Mapping) else None - if value is None: - value = model.get("input_modalities", model.get("modalities")) - if not isinstance(value, list) or not value: + reported = architecture.get("input_modalities") if isinstance(architecture, Mapping) else None + if reported is None: + reported = model.get("input_modalities", model.get("modalities")) + if not isinstance(reported, list) or not reported: return None - if not all(isinstance(entry, str) and entry in _LLAMA_CPP_INPUT_MODALITIES for entry in value): + if not all(isinstance(entry, str) and entry in _KNOWN_INPUT_MODALITIES for entry in reported): return None - supported = tuple(dict.fromkeys(entry for entry in value if entry in {"text", "image"})) - return cast(tuple[Literal["text", "image"], ...], supported) or None + sent = tuple(entry for entry in _SENT_INPUT_MODALITIES if entry in reported) + return cast(InputModalities, sent) or None -def props_input_modalities(props: object) -> tuple[Literal["text", "image"], ...] | None: - """Return the inputs a standard server's ``/props`` reports as ``modalities.vision``.""" +def props_input_modalities(props: object) -> InputModalities | None: + """Return the inputs that ``modalities.vision`` in a standard server's ``/props`` reports.""" modalities = props.get("modalities") if isinstance(props, Mapping) else None vision = modalities.get("vision") if isinstance(modalities, Mapping) else None if not isinstance(vision, bool): diff --git a/src/tau_coding/extensions/builtins/llama_cpp/service.py b/src/tau_coding/extensions/builtins/llama_cpp/service.py index 60b270e82..1cf7f6c02 100644 --- a/src/tau_coding/extensions/builtins/llama_cpp/service.py +++ b/src/tau_coding/extensions/builtins/llama_cpp/service.py @@ -325,7 +325,8 @@ async def discover( payload = _json_object(response, "/v1/models") models = _parse_models(payload) if len(models) == 1 and models[0].input_modalities is None: - models = (await self._with_props_modalities(client, headers, models[0]),) + # Older llama.cpp builds report vision only in /props. + models = (await self._inputs_from_props(client, headers, models[0]),) return LlamaCppDiscovery( self.endpoint, models, @@ -1443,17 +1444,17 @@ async def _detect_router_safely( ), ) - async def _with_props_modalities( + async def _inputs_from_props( self, client: httpx.AsyncClient, headers: Mapping[str, str], model: ProviderModel, ) -> ProviderModel: - """Fill a single model's inputs from ``/props`` when ``/v1/models`` omits them. + """Return ``model`` with the inputs that ``/props`` reports, if it reports any. - llama.cpp builds before October 2026 report image support only in - ``/props``, which describes the one loaded model of a standard server. - Any failure leaves the model unchanged. + llama.cpp builds before October 2026 report vision only in ``/props``. + On a standard server it describes the one loaded model. A failed or + unusable response leaves the model unchanged. """ try: response = await client.get(self.endpoint.server_root + "/props", headers=dict(headers)) diff --git a/tests/test_llama_cpp_extension.py b/tests/test_llama_cpp_extension.py index 382cd704b..761c3f80a 100644 --- a/tests/test_llama_cpp_extension.py +++ b/tests/test_llama_cpp_extension.py @@ -419,6 +419,7 @@ def handler(request: httpx.Request) -> httpx.Response: ({"modalities": ["text"]}, ("text",)), ({"architecture": {"input_modalities": ["text", "unknown"]}}, None), ({"architecture": {"input_modalities": ["audio"]}}, None), + ({"architecture": {"input_modalities": ["text", {"kind": "image"}]}}, None), ({"architecture": {"input_modalities": []}}, None), ({"architecture": "text"}, None), ({}, None), From 4890e1526c4434bcbec9704e4fe8ff1fa463058a Mon Sep 17 00:00:00 2001 From: Alejandro Date: Wed, 14 Oct 2026 10:24:02 +0200 Subject: [PATCH 3/3] docs(llama-cpp): explain props fallback and use Astro catalog link --- website/content/guides/local-inference.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/website/content/guides/local-inference.md b/website/content/guides/local-inference.md index f8d2ea10d..b0d992b3b 100644 --- a/website/content/guides/local-inference.md +++ b/website/content/guides/local-inference.md @@ -130,9 +130,10 @@ Image input follows what the server reports. A model whose `/v1/models` entry lists `image` in `architecture.input_modalities` (llama.cpp reports this when it was started with a vision projector, `--mmproj`) receives images, for example from the read tool. Reported `audio` and `video` inputs are ignored because Tau -does not send them. A server that reports no modalities, such as an older build -or a fork, is treated as text-only; declare its image input with a -[`catalog.toml`]({{< relref "./providers-and-models.md" >}}) entry instead. +does not send them. For older single-model servers that report no usable inputs +in `/v1/models`, Tau checks `/props.modalities.vision`. If neither endpoint +reports usable capabilities, the model is treated as text-only; declare its +image input with a [`catalog.toml`](/guides/providers-and-models/) entry instead. The safe integration snapshot is stored at `~/.tau/state/extensions/llama.cpp.json`. It contains only the normalized