diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 7fe1902405..1ec9eec6a9 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -105,29 +105,38 @@ jobs: - name: Download NLTK data run: uv run python -m nltk.downloader punkt_tab - name: Install Ollama - # Pinned to a GCC-13 build to fix a segfault loading granite4.1:3b on - # AMX-capable ubuntu-latest runners (#1388). Ollama's Linux builds through - # 0.32.1 were compiled with GCC 11, which miscompiles the AMX CPU kernels in - # libggml-cpu-sapphirerapids.so -- so ~1/3 of runs (the AMX-capable runner - # draws) crashed during model warmup, independent of ollama version and - # unreachable by any env var (the code path is correct; the machine code was - # not). Ollama PR #17244 ("bump Linux toolchain to GCC 13", Fixes ollama#17006 - # / #17205) first shipped in the 0.32.2 line. The rc0 tag has since been - # removed (promoted to the v0.32.2 release), so its download URL now 404s and - # the pin is bumped to 0.32.2. Drop this pin once a stable (non-prerelease) - # release includes the GCC-13 build. NB: OLLAMA_VERSION must omit the leading - # "v" -- install.sh prepends it (a "v" yields a broken .../download/vv0.32.2/... - # path). - run: curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.32.2 sh + # GCC-13 build: fixes a segfault loading Granite models on AMX-capable + # runners (Ollama miscompiled AMX kernels through 0.32.1, #1388). + # Pinned to 0.33.1 (current stable with the fix); also required for + # the reasoning_effort="none" thinking-off support used below. + # NB: OLLAMA_VERSION must omit the leading "v" (install.sh prepends it). + run: curl -fsSL https://ollama.com/install.sh | OLLAMA_VERSION=0.33.1 sh - name: Start serving ollama - run: nohup ollama serve & + # Debug + per-request logging, dumped on failure below: needed to tell + # "server hung" from "bad request" when a live-model test stalls. + run: OLLAMA_DEBUG=1 OLLAMA_DEBUG_LOG_REQUESTS=1 nohup ollama serve > /tmp/ollama-serve.log 2>&1 & + - name: Start resource sampler + # Memory/swap/CPU every 30s, dumped on failure below — rules out + # memory pressure as a cause of live-model stalls. + run: | + if command -v vmstat >/dev/null; then + (vmstat 30 480) > /tmp/ci-resource-sampler.log 2>&1 & + else + (for i in $(seq 1 480); do + echo "sample $i $(date -u +%FT%TZ)" + free -m + cat /proc/loadavg + echo --- + sleep 30 + done) > /tmp/ci-resource-sampler.log 2>&1 & + fi - name: Pull models # granite-vision-4.1 is not in the Ollama library, so it is pulled from # IBM's official GGUF repo on Hugging Face (that build ships the mmproj # projector Ollama needs for image input). Drop the hf.co prefix if the # model is ever published to the Ollama library directly. run: | - for model in granite4.1:3b hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M; do + for model in granite4.2:3b granite4:micro-h hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M; do pulled=false for i in 1 2 3 4 5; do ollama pull "$model" && { pulled=true; break; } @@ -144,6 +153,20 @@ jobs: # The names below are what the tests match against; print them so a # tag-name mismatch is diagnosable from the log alone. ollama list + - name: Constrain the granite4.2:3b context for the CI runner + # The published tag ships num_ctx=131072, a ~6 GB KV cache at load — + # too much for the 16 GB runner and a cause of the CI stalls. Re-point + # the local tag at an 8192-context build of the same weights (same tag + # name, so no test code changes; 2x what other Ollama tests use). + run: | + ollama cp granite4.2:3b granite4.2:3b-128k + printf 'FROM granite4.2:3b-128k\nPARAMETER num_ctx 8192\n' \ + > /tmp/MODELFILE-granite42-ci + ollama create granite4.2:3b -f /tmp/MODELFILE-granite42-ci + # grep exits non-zero (failing the step) if num_ctx did not land at + # exactly 8192 — matching only '^PARAMETER' would still pass on the + # base tag's untouched 131072 value if `ollama create` silently no-oped. + ollama show granite4.2:3b --modelfile | grep -qx 'PARAMETER num_ctx 8192' - name: Run Tests id: tests env: @@ -156,6 +179,31 @@ jobs: - name: Send failure message tests if: failure() # This step will only run if a previous step failed run: echo "Tests failed. Please verify that tests are working locally." + - name: Dump Ollama server log on failure + if: failure() # Only the test step can produce a useful server log here. + # Raw log is too verbose to dump whole; extract requests/errors/model + # events plus a short raw tail. + run: | + echo "=== ollama serve log: requests (GIN), errors, model events ===" + grep -iE 'GIN|error|warn|panic|loading|unload' /tmp/ollama-serve.log | tail -n 500 || echo "(no matching lines)" + echo "=== ollama serve log (last 50 raw lines) ===" + tail -n 50 /tmp/ollama-serve.log || echo "(no ollama serve log found)" + echo "=== resource sampler (30 s: memory, swap si/so, CPU) ===" + cat /tmp/ci-resource-sampler.log || echo "(no sampler log found)" + echo "=== ollama request debug logs (request bodies + replay curls) ===" + reqdir=$(grep -oE '/tmp/ollama-request-logs-[0-9a-z]+' /tmp/ollama-serve.log | head -1) + if [ -n "$reqdir" ] && [ -d "$reqdir" ]; then + echo "file count: $(find "$reqdir" -maxdepth 1 -type f | wc -l)" + # Newest 20 files (mtime-descending, "%T@" epoch + path). + find "$reqdir" -maxdepth 1 -type f -printf '%T@ %p\n' \ + | sort -rn | head -20 | while IFS= read -r line; do + f=${line#* } + echo "--- $f ---" + cat "$f" + done + else + echo "(no request debug dir found in serve log)" + fi - name: Write job summary if: always() run: | diff --git a/docs/docs/examples/index.md b/docs/docs/examples/index.md index d65cc9665f..14bf81a7a0 100644 --- a/docs/docs/examples/index.md +++ b/docs/docs/examples/index.md @@ -130,4 +130,4 @@ uv run docs/examples//.py **Default backend:** `start_session()` with no arguments connects to a local [Ollama](https://ollama.ai) instance running **IBM Granite 4 Micro** -(`granite4.1:3b`). Make sure Ollama is running before you execute any example. +(`granite4.2:3b`). Make sure Ollama is running before you execute any example. diff --git a/docs/docs/getting-started/installation.md b/docs/docs/getting-started/installation.md index 14e6000273..a00260b1d6 100644 --- a/docs/docs/getting-started/installation.md +++ b/docs/docs/getting-started/installation.md @@ -62,5 +62,5 @@ The default session connects to [Ollama](https://ollama.ai) running locally. Install Ollama and pull the default model before running any examples: ```bash -ollama pull granite4.1:3b +ollama pull granite4.2:3b ``` diff --git a/docs/docs/getting-started/quickstart.md b/docs/docs/getting-started/quickstart.md index d4c1fb6470..3e4bacd48a 100644 --- a/docs/docs/getting-started/quickstart.md +++ b/docs/docs/getting-started/quickstart.md @@ -10,7 +10,7 @@ description: "Run your first generative program in minutes." ## Hello world By default, `start_session()` connects to Ollama and uses **IBM Granite 4 Micro** -(`granite4.1:3b`). Make sure Ollama is running before you run this: +(`granite4.2:3b`). Make sure Ollama is running before you run this: ```python import mellea @@ -205,7 +205,7 @@ Hugging Face, and WatsonX are also supported. See ## Troubleshooting -**`granite4.1:3b` not found** — run `ollama pull granite4.1:3b` before starting. +**`granite4.2:3b` not found** — run `ollama pull granite4.2:3b` before starting. **Python 3.13 `outlines` install failure** — `outlines` requires a Rust compiler. Either install [Rust](https://www.rust-lang.org/tools/install) or pin Python to 3.12. diff --git a/docs/docs/troubleshooting/common-errors.md b/docs/docs/troubleshooting/common-errors.md index 8248c4a735..b867892450 100644 --- a/docs/docs/troubleshooting/common-errors.md +++ b/docs/docs/troubleshooting/common-errors.md @@ -6,16 +6,16 @@ description: "Common errors, diagnostic steps, and fixes for Mellea programs." ## Installation -### `granite4.1:3b` not found +### `granite4.2:3b` not found ```text -Error: model "granite4.1:3b" not found +Error: model "granite4.2:3b" not found ``` Pull the model before running: ```bash -ollama pull granite4.1:3b +ollama pull granite4.2:3b ``` ### Python 3.13: `outlines` install failure diff --git a/docs/docs/troubleshooting/faq.md b/docs/docs/troubleshooting/faq.md index 4170fa4af5..5fd79bec60 100644 --- a/docs/docs/troubleshooting/faq.md +++ b/docs/docs/troubleshooting/faq.md @@ -38,7 +38,7 @@ m = MelleaSession( ) ``` -## How do I use a model other than `granite4.1:3b`? +## How do I use a model other than `granite4.2:3b`? Pass the `model_id` parameter to `start_session()`: diff --git a/docs/docs/tutorials/01-your-first-generative-program.md b/docs/docs/tutorials/01-your-first-generative-program.md index ccfeeecf67..0cece77cfe 100644 --- a/docs/docs/tutorials/01-your-first-generative-program.md +++ b/docs/docs/tutorials/01-your-first-generative-program.md @@ -20,7 +20,7 @@ By the end you will have covered: > see [Tutorial 03: Using Generative Stubs](../tutorials/using-generative-stubs). **Prerequisites:** [Quick Start](../getting-started/quickstart) complete, -Mellea installed (`uv add mellea`), Ollama running locally with `granite4.1:3b` downloaded. +Mellea installed (`uv add mellea`), Ollama running locally with `granite4.2:3b` downloaded. --- diff --git a/docs/docs/tutorials/02-streaming-and-async.md b/docs/docs/tutorials/02-streaming-and-async.md index 9844163a1c..b9f27eaeeb 100644 --- a/docs/docs/tutorials/02-streaming-and-async.md +++ b/docs/docs/tutorials/02-streaming-and-async.md @@ -16,7 +16,7 @@ By the end you will have covered: - Context behaviour with concurrent async calls **Prerequisites:** [Tutorial 01](./your-first-generative-program) complete, -`pip install mellea`, Ollama running locally with `granite4.1:3b` downloaded. +`pip install mellea`, Ollama running locally with `granite4.2:3b` downloaded. --- diff --git a/docs/docs/tutorials/03-using-generative-stubs.md b/docs/docs/tutorials/03-using-generative-stubs.md index 74f049581e..5ee0356cd4 100644 --- a/docs/docs/tutorials/03-using-generative-stubs.md +++ b/docs/docs/tutorials/03-using-generative-stubs.md @@ -16,7 +16,7 @@ By the end you will have covered: - Precondition and postcondition validation patterns **Prerequisites:** [Tutorial 01](./your-first-generative-program) complete, -`pip install mellea`, Ollama running locally with `granite4.1:3b` downloaded. +`pip install mellea`, Ollama running locally with `granite4.2:3b` downloaded. --- diff --git a/mellea/backends/litellm.py b/mellea/backends/litellm.py index 061aa516aa..c20f42348a 100644 --- a/mellea/backends/litellm.py +++ b/mellea/backends/litellm.py @@ -76,7 +76,10 @@ class LiteLLMBackend(FormatterBackend): `ollama_chat/` → localhost:11434, `anthropic/` → Anthropic API). Use `None` for cloud providers; set explicitly for local servers such as vLLM or a non-default Ollama port. - model_options (dict | None): Default model options for generation requests. + model_options (dict | None): Default model options applied to every + generation request. Per-call options take precedence. Use + `{ModelOption.THINKING: False}` here to suppress the think block + on models that enable it by default. Attributes: to_mellea_model_opts_map (dict): Mapping from backend-specific option names to @@ -87,7 +90,7 @@ class LiteLLMBackend(FormatterBackend): def __init__( self, - model_id: str = "ollama_chat/" + str(model_ids.IBM_GRANITE_4_1_3B.ollama_name), + model_id: str = "ollama_chat/" + str(model_ids.IBM_GRANITE_4_2_3B.ollama_name), formatter: ChatFormatter | None = None, base_url: str | None = None, model_options: dict | None = None, @@ -384,9 +387,9 @@ async def _generate_from_chat_context_standard( # Map THINKING to the correct backend parameter(s). Two mechanisms: # - chat_template_kwargs.enable_thinking: vLLM/Qwen3/Gemma4 (bool toggle) - # - reasoning_effort: LiteLLM/OpenAI-compatible (string level, or True → "medium") - # Both are set for True so each server picks up whichever it understands. - # NOTE: don't pass reasoning_effort=False — it is invalid; absence disables reasoning. + # - reasoning_effort: LiteLLM/OpenAI-compatible (string level; True → "medium", + # False → "none") + # Both are set so each server picks up whichever it understands. thinking = model_opts.get(ModelOption.THINKING, None) original_thinking = thinking # preserve raw caller value for the generate log reasoning_params: dict[str, Any] = {} @@ -399,8 +402,15 @@ async def _generate_from_chat_context_standard( extra_params["extra_body"] = ctk_body if thinking: reasoning_params["reasoning_effort"] = "medium" - # False: do not send reasoning_effort — absent param disables reasoning; - # passing False would be invalid. + elif "ollama" in self._model_id.split("/")[0]: + # Ollama-served thinking models (e.g. granite4.2) default to + # thinking ON when reasoning_effort is absent; "none" is the + # OpenAI enum value their /v1 endpoint maps to think=false + # (Ollama >= 0.33.1). Real OpenAI/other reasoning providers + # reject "none", so this is scoped to Ollama-routed models + # (same provider-prefix check used for the streaming + # tool-call workaround above). + reasoning_params["reasoning_effort"] = "none" else: reasoning_params["reasoning_effort"] = thinking diff --git a/mellea/backends/model_ids.py b/mellea/backends/model_ids.py index 06ab71b090..e3a454763c 100644 --- a/mellea/backends/model_ids.py +++ b/mellea/backends/model_ids.py @@ -115,6 +115,25 @@ class ModelIdentifier: context_length=131072, ) +# Granite 4.2 Dense Models +IBM_GRANITE_4_2_3B = ModelIdentifier( + hf_model_name="ibm-granite/granite-4.2-3b", + ollama_name="granite4.2:3b", + context_length=131072, +) + +IBM_GRANITE_4_2_8B = ModelIdentifier( + hf_model_name="ibm-granite/granite-4.2-8b", + ollama_name="granite4.2:8b", + context_length=131072, +) + +IBM_GRANITE_4_2_30B = ModelIdentifier( + hf_model_name="ibm-granite/granite-4.2-30b", + ollama_name="granite4.2:30b", + context_length=131072, +) + IBM_GRANITE_GUARDIAN_4_1_8B = ModelIdentifier( hf_model_name="ibm-granite/granite-guardian-4.1-8b", context_length=131072 ) diff --git a/mellea/backends/model_options.py b/mellea/backends/model_options.py index 18d8fe063f..1a61948bde 100644 --- a/mellea/backends/model_options.py +++ b/mellea/backends/model_options.py @@ -65,8 +65,11 @@ class ModelOption: silently ignored by those servers. * `False` — native Ollama backend: sends `think=False`. OpenAI-compatible backends: sets `chat_template_kwargs.enable_thinking=False` - to suppress the think block. `reasoning_effort` is not sent (passing - `False` would be an invalid value for OpenAI; absence disables reasoning). + and `reasoning_effort="none"` to suppress the think block. Both are sent so + each server type picks up the mechanism it understands: vLLM honours + `chat_template_kwargs`, while Ollama's /v1 endpoint (>= 0.33.1) honours + `reasoning_effort` — and for models that default to thinking on (e.g. + granite4.2) absent `reasoning_effort` means thinking stays on. * `"low"` / `"medium"` / `"high"` — passed directly as `reasoning_effort` (OpenAI-compatible backends only; no-op on vLLM). diff --git a/mellea/backends/ollama.py b/mellea/backends/ollama.py index 2babc396ce..c991d10a10 100644 --- a/mellea/backends/ollama.py +++ b/mellea/backends/ollama.py @@ -174,7 +174,7 @@ class OllamaModelBackend(FormatterBackend): def __init__( self, - model_id: str | ModelIdentifier = model_ids.IBM_GRANITE_4_1_3B, + model_id: str | ModelIdentifier = model_ids.IBM_GRANITE_4_2_3B, formatter: ChatFormatter | None = None, base_url: str | None = None, model_options: dict | None = None, diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 739f5581b7..446a764bf4 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -519,6 +519,47 @@ def _make_backend_specific_and_remove( return model_opts + def _map_thinking_option( + self, thinking: Any, extra_body: dict[str, Any] + ) -> dict[str, Any]: + """Maps `ModelOption.THINKING` to the correct backend parameter(s). + + Two mechanisms, both set (when applicable) so the right server picks + up whichever it understands: + - `extra_body["chat_template_kwargs"]["enable_thinking"]`: vLLM/Qwen3 + - `reasoning_effort`: OpenAI/DeepSeek/Ollama (string level; True → + "medium", False → "none") + + Ollama-served models (e.g. granite4.2) think by default unless + `reasoning_effort="none"` is sent (Ollama >= 0.33.1); real OpenAI + rejects `"none"`, so that value is scoped to non-OpenAI servers. + + Args: + thinking: the raw `ModelOption.THINKING` value (bool, string + reasoning-effort level, or None). + extra_body: the in-progress `extra_body` dict for this request; + mutated in place to add `chat_template_kwargs` if `thinking` + is a bool. + + Returns: + dict[str, Any]: `reasoning_effort` params to merge into the + request's top-level kwargs, or `{}` if `thinking` is None. + """ + reasoning_params: dict[str, Any] = {} + if thinking is None: # False is a valid value — cannot use `if thinking` + return reasoning_params + if type(thinking) is bool: + ctk = extra_body.get("chat_template_kwargs", {}) or {} + ctk["enable_thinking"] = thinking + extra_body["chat_template_kwargs"] = ctk + if thinking: + reasoning_params["reasoning_effort"] = "medium" + elif self._server_type != _ServerType.OPENAI: + reasoning_params["reasoning_effort"] = "none" + else: + reasoning_params["reasoning_effort"] = thinking + return reasoning_params + def _merge_user_extra_body( self, base: dict[str, Any], user: dict[str, Any] | None ) -> dict[str, Any]: @@ -809,22 +850,8 @@ async def _generate_from_intrinsic( user_extra_body = user_api_params.pop("extra_body", None) api_params.update(user_api_params) - # Map THINKING to the correct backend parameter(s). Two mechanisms: - # - chat_template_kwargs.enable_thinking: vLLM/Qwen3 (bool toggle) - # - reasoning_effort: OpenAI/DeepSeek (string level, or True → "medium") - # Both are set for True so the right server picks up whichever it understands. thinking = model_options.get(ModelOption.THINKING) - if thinking is not None: # False is a valid value — cannot use `if thinking` - if type(thinking) is bool: - ctk = extra_body.get("chat_template_kwargs", {}) or {} - ctk["enable_thinking"] = thinking - extra_body["chat_template_kwargs"] = ctk - if thinking: - api_params["reasoning_effort"] = "medium" - # False: don't send reasoning_effort — OpenAI disables reasoning by - # default when the param is absent; passing False would be invalid. - else: - api_params["reasoning_effort"] = thinking + api_params.update(self._map_thinking_option(thinking, extra_body)) extra_body = self._merge_user_extra_body(extra_body, user_extra_body) @@ -1048,25 +1075,11 @@ async def _generate_from_chat_context_standard( formatted_tools = convert_tools_to_json(tools) use_tools = len(formatted_tools) > 0 - # Map THINKING to the correct backend parameter(s). Two mechanisms: - # - chat_template_kwargs.enable_thinking: vLLM/Qwen3 (bool toggle) - # - reasoning_effort: OpenAI/DeepSeek (string level, or True → "medium") - # NOTE: don't pass reasoning_effort to non-reasoning models (e.g. gpt-4o). + # NOTE: don't pass THINKING to non-reasoning models (e.g. gpt-4o). thinking = model_opts.get(ModelOption.THINKING) - reasoning_params: dict[str, Any] = {} - if thinking is not None: # False is a valid value — cannot use `if thinking` - if type(thinking) is bool: - ctk_body: dict[str, Any] = extra_params.get("extra_body", {}) or {} - ctk = ctk_body.get("chat_template_kwargs", {}) or {} - ctk["enable_thinking"] = thinking - ctk_body["chat_template_kwargs"] = ctk - extra_params["extra_body"] = ctk_body - if thinking: - reasoning_params["reasoning_effort"] = "medium" - # False: don't send reasoning_effort — OpenAI disables reasoning by - # default when the param is absent; passing False would be invalid. - else: - reasoning_params["reasoning_effort"] = thinking + ctk_body: dict[str, Any] = extra_params.get("extra_body", {}) or {} + reasoning_params = self._map_thinking_option(thinking, ctk_body) + extra_params["extra_body"] = ctk_body # Request usage information in streaming responses if model_opts.get(ModelOption.STREAM, False): diff --git a/mellea/formatters/granite/intrinsics/constants.py b/mellea/formatters/granite/intrinsics/constants.py index 7f3b5c923f..87963c4efa 100644 --- a/mellea/formatters/granite/intrinsics/constants.py +++ b/mellea/formatters/granite/intrinsics/constants.py @@ -37,10 +37,16 @@ "ibm-granite/granite-4.1-3b": "granite-4.1-3b", "ibm-granite/granite-4.1-8b": "granite-4.1-8b", "ibm-granite/granite-4.1-30b": "granite-4.1-30b", + "ibm-granite/granite-4.2-3b": "granite-4.2-3b", + "ibm-granite/granite-4.2-8b": "granite-4.2-8b", + "ibm-granite/granite-4.2-30b": "granite-4.2-30b", "granite4:micro": "granite4_micro", "granite4.1:3b": "granite4.1_3b", "granite4.1:8b": "granite4.1_8b", "granite4.1:30b": "granite4.1_30b", + "granite4.2:3b": "granite4.2_3b", + "granite4.2:8b": "granite4.2_8b", + "granite4.2:30b": "granite4.2_30b", } """Base model names that we accept for LoRA/aLoRA adapters in intrinsics libraries. Each model name maps to the name of the directory that contains (a)LoRA adapters for diff --git a/mellea/stdlib/session.py b/mellea/stdlib/session.py index 303572aaeb..146d47252b 100644 --- a/mellea/stdlib/session.py +++ b/mellea/stdlib/session.py @@ -23,7 +23,7 @@ from PIL import Image as PILImage from ..backends.model_ids import ( - IBM_GRANITE_4_1_3B, + IBM_GRANITE_4_2_3B, IBM_GRANITE_4_HYBRID_SMALL, ModelIdentifier, ) @@ -94,7 +94,7 @@ def get_session() -> MelleaSession: def start_session( backend_name: Literal["ollama", "hf", "openai", "watsonx", "litellm"] = "ollama", - model_id: str | ModelIdentifier = IBM_GRANITE_4_1_3B, + model_id: str | ModelIdentifier = IBM_GRANITE_4_2_3B, ctx: Context | None = None, *, context_type: Literal["simple", "chat"] | None = None, diff --git a/test/backends/conftest.py b/test/backends/conftest.py index ce5b20e2e5..fca330cb74 100644 --- a/test/backends/conftest.py +++ b/test/backends/conftest.py @@ -34,7 +34,7 @@ def test_something(mock_ollama_backend): """ def _make( - model_id: str = "granite4.1:3b", + model_id: str = "granite4.2:3b", model_options: dict | None = None, timeout: float | None = None, ) -> OllamaModelBackend: diff --git a/test/backends/test_acall_tools_parallel_execution.py b/test/backends/test_acall_tools_parallel_execution.py index 12983625b5..ff82cddd85 100644 --- a/test/backends/test_acall_tools_parallel_execution.py +++ b/test/backends/test_acall_tools_parallel_execution.py @@ -17,7 +17,6 @@ import pytest -from mellea.backends.ollama import OllamaModelBackend from mellea.backends.tools import MelleaTool from mellea.core.base import ModelOutputThunk, ModelToolCall from mellea.stdlib.functional import acall_tools @@ -25,14 +24,14 @@ pytestmark = [pytest.mark.integration] -@pytest.fixture(scope="module") -def backend(): +@pytest.fixture +def backend(mock_ollama_backend): """Create an OllamaModelBackend for formatter.print() only. Note: acall_tools() only uses backend.formatter, not inference. Tests use local Python functions as tool implementations, no model calls. """ - return OllamaModelBackend() + return mock_ollama_backend() @pytest.mark.asyncio diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 11ba5a2246..c9573462c3 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -57,11 +57,8 @@ def backend(): """Shared HuggingFace backend for all tests in this module. - Uses Granite 3.3-8b for aLoRA adapter compatibility. - Note: as of #1135, the "requirement-check" intrinsic catalogue entry points to - granitelib-core-r1.0 (granite-4.x adapters). Tests that exercise requirement-check - against this granite-3.3 backend will fail once revision pinning is wired through - in phase-2.2 (#1141). Other intrinsics are not affected. + Uses Granite 4.1 3B because the pinned adapter-function catalogues do not + provide Granite 4.2 weights yet. """ with hf_skip(): backend = LocalHFBackend( diff --git a/test/backends/test_litellm_ollama.py b/test/backends/test_litellm_ollama.py index aec4eaf415..0136a7e307 100644 --- a/test/backends/test_litellm_ollama.py +++ b/test/backends/test_litellm_ollama.py @@ -18,7 +18,8 @@ from mellea.stdlib.context import SimpleContext from mellea.stdlib.sampling import RejectionSamplingStrategy -_MODEL_ID = f"ollama_chat/{model_ids.IBM_GRANITE_4_1_3B.ollama_name}" +assert model_ids.IBM_GRANITE_4_2_3B.ollama_name is not None +_MODEL_ID = f"ollama_chat/{model_ids.IBM_GRANITE_4_2_3B.ollama_name}" @pytest.fixture(scope="function") diff --git a/test/backends/test_litellm_thinking.py b/test/backends/test_litellm_thinking.py index c4bde886af..602b30088b 100644 --- a/test/backends/test_litellm_thinking.py +++ b/test/backends/test_litellm_thinking.py @@ -300,13 +300,51 @@ async def test_thinking_true_sets_reasoning_effort_and_enable_thinking( async def test_thinking_false_omits_reasoning_effort_and_sets_disable( chat_backend: LiteLLMBackend, ) -> None: - """THINKING=False: reasoning_effort absent, extra_body.chat_template_kwargs.enable_thinking=False.""" + """THINKING=False on a non-Ollama target: reasoning_effort absent, chat_template_kwargs.enable_thinking=False. + + `reasoning_effort="none"` is only accepted by newer OpenAI reasoning + models and is an Ollama-/v1-specific workaround (Ollama >= 0.33.1 maps + it to think=false; see test_thinking_false_sets_reasoning_effort_none_for_ollama + below). Sending it to a real OpenAI or vLLM target — like this + `hosted_vllm/qwen3` fixture — can turn a previously-working call into a + 400. vLLM already honours chat_template_kwargs.enable_thinking, so that + mechanism alone is sufficient here. + """ from mellea.backends import ModelOption kwargs = await _call_and_capture(chat_backend, {ModelOption.THINKING: False}) assert "reasoning_effort" not in kwargs, ( - "reasoning_effort must not be sent for THINKING=False (invalid value)" + "reasoning_effort must not be sent for THINKING=False on a non-Ollama target" + ) + assert ( + kwargs.get("extra_body", {}) + .get("chat_template_kwargs", {}) + .get("enable_thinking") + is False + ), "extra_body.chat_template_kwargs.enable_thinking should be False" + + +async def test_thinking_false_sets_reasoning_effort_none_for_ollama() -> None: + """THINKING=False on an Ollama-routed target sets reasoning_effort='none'. + + Ollama's /v1 endpoint (>= 0.33.1) honours reasoning_effort — where + absence means the model default (thinking ON for e.g. granite4.2), so + "none" is required to actually disable the think block. This is scoped + to Ollama-routed model IDs (see the "ollama" in self._model_id.split("/")[0] + check, which mirrors the streaming tool-call workaround elsewhere in + LiteLLMBackend) so it doesn't reach real OpenAI or vLLM targets. + """ + from mellea.backends import ModelOption + + backend = LiteLLMBackend( + model_id="ollama_chat/granite4.2:3b", base_url="http://localhost:11434" + ) + kwargs = await _call_and_capture(backend, {ModelOption.THINKING: False}) + + assert kwargs.get("reasoning_effort") == "none", ( + "reasoning_effort should be 'none' for THINKING=False " + "(disables thinking on Ollama /v1, which defaults it on)" ) assert ( kwargs.get("extra_body", {}) diff --git a/test/backends/test_model_ids.py b/test/backends/test_model_ids.py index 6b3301af6d..b7e2fa8034 100644 --- a/test/backends/test_model_ids.py +++ b/test/backends/test_model_ids.py @@ -12,7 +12,10 @@ import pytest import mellea.backends.model_ids as model_ids +from mellea.backends.litellm import LiteLLMBackend from mellea.backends.model_ids import ModelIdentifier +from mellea.backends.ollama import OllamaModelBackend +from mellea.stdlib.session import start_session # Collect all ModelIdentifier constants defined at module level. _ALL_IDS: list[tuple[str, ModelIdentifier]] = [ @@ -29,6 +32,36 @@ ] +@pytest.mark.parametrize( + ("model_id", "expected_ollama_name"), + [ + (model_ids.IBM_GRANITE_4_2_3B, "granite4.2:3b"), + (model_ids.IBM_GRANITE_4_2_8B, "granite4.2:8b"), + (model_ids.IBM_GRANITE_4_2_30B, "granite4.2:30b"), + ], +) +def test_granite_4_2_ollama_names( + model_id: ModelIdentifier, expected_ollama_name: str +) -> None: + """Granite 4.2 identifiers use the published Ollama library tags.""" + assert model_id.ollama_name == expected_ollama_name + + +def test_ollama_defaults_use_granite_4_2() -> None: + """Default local backends and sessions use Granite 4.2 3B.""" + granite_4_2 = model_ids.IBM_GRANITE_4_2_3B + assert ( + inspect.signature(OllamaModelBackend).parameters["model_id"].default + is granite_4_2 + ) + assert inspect.signature(LiteLLMBackend).parameters["model_id"].default == ( + f"ollama_chat/{granite_4_2.ollama_name}" + ) + assert ( + inspect.signature(start_session).parameters["model_id"].default is granite_4_2 + ) + + @pytest.mark.integration @pytest.mark.slow @pytest.mark.parametrize("const_name,hf_name", _HF_IDS, ids=[n for n, _ in _HF_IDS]) diff --git a/test/backends/test_ollama.py b/test/backends/test_ollama.py index 1e25d8b06f..d96eaeceea 100644 --- a/test/backends/test_ollama.py +++ b/test/backends/test_ollama.py @@ -3,6 +3,7 @@ import asyncio import json +import os from typing import Annotated import ollama as _ollama @@ -10,8 +11,8 @@ import pytest from mellea import start_session -from mellea.backends import ModelOption -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea.backends import ModelOption, model_ids +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B from mellea.backends.ollama import OllamaModelBackend from mellea.core import CBlock, Requirement from mellea.stdlib.context import SimpleContext @@ -20,10 +21,32 @@ # Mark all tests in this module as requiring Ollama pytestmark = [pytest.mark.ollama, pytest.mark.e2e] +# Match granite4.2:3b's constrained default (Modelfile num_ctx: 8192) so the +# runner is loaded once and never reloaded for a context-size mismatch. +TEST_CONTEXT_WINDOW = 8192 + + +def _ollama_model_for_eval() -> str: + """Return the Ollama model tag driven by GRANITE42_MODEL env var. + + Accepts either an Ollama tag (granite4.2:8b) or an HF model ID + (ibm-granite/granite-4.2-8b) — both select the right size. + Defaults to 3B. + """ + name = os.environ.get("GRANITE42_MODEL", "") + if "8b" in name or "8B" in name: + assert model_ids.IBM_GRANITE_4_2_8B.ollama_name is not None + return model_ids.IBM_GRANITE_4_2_8B.ollama_name + if "30b" in name or "30B" in name: + assert model_ids.IBM_GRANITE_4_2_30B.ollama_name is not None + return model_ids.IBM_GRANITE_4_2_30B.ollama_name + assert IBM_GRANITE_4_2_3B.ollama_name is not None + return IBM_GRANITE_4_2_3B.ollama_name + @pytest.fixture(scope="module", autouse=True) def _ensure_model_warm() -> None: - """Warm up the default model before tests run in this module. + """Warm up the selected model before tests run in this module. The conftest warms models when transitioning *into* the ollama test group, but that warm-up does not fire when this file is run in isolation (e.g. @@ -34,12 +57,17 @@ def _ensure_model_warm() -> None: `keep_alive=-1` pins the model in memory until the conftest module-boundary eviction fires at the end of this test file. + + Set GRANITE42_MODEL=granite4.2:8b (or ibm-granite/granite-4.2-8b) + to warm a different size. """ - _model = IBM_GRANITE_4_1_3B.ollama_name - assert _model is not None # IBM_GRANITE_4_1_3B always has ollama_name set + _model = _ollama_model_for_eval() try: _ollama.generate( - model=_model, prompt="hi", options={"num_predict": 1}, keep_alive=-1 + model=_model, + prompt="hi", + options={"num_ctx": TEST_CONTEXT_WINDOW, "num_predict": 1}, + keep_alive=-1, ) except Exception: pass # best-effort; per-test failures will be clearer than a fixture abort @@ -47,8 +75,17 @@ def _ensure_model_warm() -> None: @pytest.fixture(scope="function") def session(): - """Fresh Ollama session for each test.""" - session = start_session() + """Fresh Ollama session for each test. + + The model size is driven by GRANITE42_MODEL (see _ollama_model_for_eval). + """ + session = start_session( + model_id=_ollama_model_for_eval(), + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + }, + ) yield session session.reset() @@ -140,7 +177,7 @@ async def test_generate_from_raw(session) -> None: actions=[CBlock(value=prompt) for prompt in prompts], ctx=session.ctx, model_options={ - ModelOption.CONTEXT_WINDOW: 2048, + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, # With raw prompts and high temperature, a response of arbitrary # length is normal operation. ModelOption.MAX_NEW_TOKENS: 100, @@ -165,7 +202,7 @@ class Answer(pydantic.BaseModel): actions=[CBlock(value=prompt) for prompt in prompts], ctx=session.ctx, format=Answer, - model_options={ModelOption.CONTEXT_WINDOW: 2048}, + model_options={ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW}, ) assert len(results) == len(prompts) diff --git a/test/backends/test_openai_intrinsics.py b/test/backends/test_openai_intrinsics.py index fb721600fb..f6fa354591 100644 --- a/test/backends/test_openai_intrinsics.py +++ b/test/backends/test_openai_intrinsics.py @@ -13,6 +13,7 @@ import pathlib import signal import subprocess +import sys import time import pytest @@ -89,6 +90,12 @@ def vllm_switch_process(): yield None return + if os.environ.get("VLLM_TEST_BASE_URL"): + pytest.skip( + "Generic vLLM server is active; Granite Switch tests require a separate server", + allow_module_level=True, + ) + # Require CUDA — vLLM does not support MPS try: subprocess.run(["nvidia-smi", "-L"], check=True, capture_output=True) @@ -100,8 +107,24 @@ def vllm_switch_process(): vllm_venv = os.environ.get("VLLM_VENV_PATH", ".vllm-venv") vllm_python = os.path.join(vllm_venv, "bin", "python") - if not os.path.isfile(vllm_python): - subprocess.run(["uv", "venv", vllm_venv, "--python", "3.11"], check=True) + expected_python = f"{sys.version_info.major}.{sys.version_info.minor}" + existing_python = ( + subprocess.run( + [ + vllm_python, + "-c", + "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ], + capture_output=True, + text=True, + ) + if os.path.isfile(vllm_python) + else None + ) + if existing_python is None or existing_python.stdout.strip() != expected_python: + subprocess.run( + ["uv", "venv", vllm_venv, "--python", sys.executable, "--clear"], check=True + ) subprocess.run( ["uv", "pip", "install", "--python", vllm_python, "vllm"], check=True ) diff --git a/test/backends/test_openai_intrinsics_unit.py b/test/backends/test_openai_intrinsics_unit.py index 3577a16dc2..660d46f14d 100644 --- a/test/backends/test_openai_intrinsics_unit.py +++ b/test/backends/test_openai_intrinsics_unit.py @@ -609,7 +609,10 @@ async def test_user_extra_body_is_not_mutated(): async def test_reasoning_effort_bool_false(): - """THINKING: False sets chat_template_kwargs.enable_thinking=False; no reasoning_effort.""" + """THINKING: False sets chat_template_kwargs.enable_thinking=False and + reasoning_effort="none" (Ollama /v1 disables thinking only via + reasoning_effort; absence means the model default, which is ON for + e.g. granite4.2).""" backend = _make_backend_with_adapter(_SIMPLE_CONFIG) ctx = _make_context() mock_create = AsyncMock(return_value=_simple_chat_completion()) @@ -633,8 +636,9 @@ async def test_reasoning_effort_bool_false(): await mot.avalue() call_kwargs = mock_create.call_args - assert "reasoning_effort" not in call_kwargs.kwargs, ( - "reasoning_effort must not be sent for THINKING=False (invalid for OpenAI)" + assert call_kwargs.kwargs.get("reasoning_effort") == "none", ( + "reasoning_effort should be 'none' for THINKING=False " + "(disables thinking on Ollama /v1, which defaults it on)" ) extra_body = call_kwargs.kwargs.get("extra_body", {}) assert extra_body.get("chat_template_kwargs", {}).get("enable_thinking") is False diff --git a/test/backends/test_openai_ollama.py b/test/backends/test_openai_ollama.py index 8ccd0698e1..1ee0defe20 100644 --- a/test/backends/test_openai_ollama.py +++ b/test/backends/test_openai_ollama.py @@ -17,7 +17,7 @@ from mellea import MelleaSession from mellea.backends import ModelOption -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B from mellea.backends.openai import OpenAIBackend from mellea.core import CBlock, ModelOutputThunk from mellea.formatters import TemplateFormatter @@ -28,10 +28,11 @@ def backend(gh_run: int): """Shared OpenAI backend configured for Ollama.""" return OpenAIBackend( - model_id=IBM_GRANITE_4_1_3B.ollama_name, # type: ignore - formatter=TemplateFormatter(model_id=IBM_GRANITE_4_1_3B.hf_model_name), # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, # type: ignore + formatter=TemplateFormatter(model_id=IBM_GRANITE_4_2_3B.hf_model_name), # type: ignore base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", api_key="ollama", + default_extra_body={"chat_template_kwargs": {"enable_thinking": False}}, ) @@ -240,10 +241,23 @@ async def get_client_async(): assert len(backend._client_cache.cache.values()) == 2 -async def test_reasoning_effort_conditional_passing(backend) -> None: +async def test_reasoning_effort_conditional_passing(gh_run: int) -> None: """Test that reasoning_effort is only passed to API when not None.""" from unittest.mock import AsyncMock, MagicMock, patch + # A dedicated backend (mocked below, never actually calls Ollama) rather + # than the shared module `backend` fixture: that fixture sets + # default_extra_body={"chat_template_kwargs": {"enable_thinking": False}}, + # which would force enable_thinking=False in every request regardless of + # what the THINKING-mapping code under test does, making the + # enable_thinking assertions below pass unconditionally. + backend = OpenAIBackend( + model_id=IBM_GRANITE_4_2_3B.ollama_name, # type: ignore + formatter=TemplateFormatter(model_id=IBM_GRANITE_4_2_3B.hf_model_name), # type: ignore + base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", + api_key="ollama", + ) + ctx = ChatContext() ctx = ctx.add(CBlock(value="Test")) @@ -296,7 +310,10 @@ async def test_reasoning_effort_conditional_passing(backend) -> None: is True ) - # Test 4: THINKING=False sets chat_template_kwargs but NOT reasoning_effort + # Test 4: THINKING=False sets both chat_template_kwargs and + # reasoning_effort="none" (Ollama /v1 disables thinking only via + # reasoning_effort; absence means the model default, which is ON for + # e.g. granite4.2) with patch.object( backend._async_client.chat.completions, "create", new_callable=AsyncMock ) as mock_create: @@ -305,8 +322,9 @@ async def test_reasoning_effort_conditional_passing(backend) -> None: CBlock(value="Hi"), ctx, model_options={ModelOption.THINKING: False} ) call_kwargs = mock_create.call_args.kwargs - assert "reasoning_effort" not in call_kwargs, ( - "reasoning_effort must not be sent for THINKING=False (invalid for OpenAI)" + assert call_kwargs.get("reasoning_effort") == "none", ( + "reasoning_effort should be 'none' for THINKING=False " + "(disables thinking on Ollama /v1, which defaults it on)" ) assert ( call_kwargs.get("extra_body", {}) diff --git a/test/backends/test_openai_unit.py b/test/backends/test_openai_unit.py index 79cb822d83..9e3f0710fc 100644 --- a/test/backends/test_openai_unit.py +++ b/test/backends/test_openai_unit.py @@ -750,5 +750,137 @@ async def test_standard_chat_path_applies_default_extra_body_without_per_call_ov assert call_kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True +async def test_thinking_true_sends_reasoning_effort_medium_and_enable_thinking(): + """THINKING=True sends reasoning_effort="medium" AND + extra_body.chat_template_kwargs.enable_thinking=True, so each server type + (OpenAI-style vs vLLM-style) picks up the mechanism it understands.""" + from mellea.core.base import CBlock + from mellea.stdlib.context import ChatContext + + backend = OpenAIBackend( + model_id="gpt-4o", base_url="http://localhost:9999/v1", api_key="test-key" + ) + + with patch.object( + backend._async_client.chat.completions, "create", new_callable=AsyncMock + ) as mock_create: + mock_create.return_value = ChatCompletion( + id="test", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(role="assistant", content="ok"), + ) + ], + created=0, + model="gpt-4o", + object="chat.completion", + ) + mot, _ = await backend.generate_from_chat_context( + CBlock(value="hello"), + ChatContext(), + model_options={ModelOption.THINKING: True}, + ) + await mot.avalue() + + call_kwargs = mock_create.call_args.kwargs + assert call_kwargs["reasoning_effort"] == "medium" + assert call_kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True + + +async def test_thinking_false_sends_reasoning_effort_none_and_disable(): + """THINKING=False sends reasoning_effort="none" AND + extra_body.chat_template_kwargs.enable_thinking=False. + + Ollama's /v1 endpoint (>= 0.33.1) maps reasoning_effort="none" to + think=false; absent the param, thinking-capable models (e.g. granite4.2) + default to thinking on. vLLM picks up the chat_template_kwargs mechanism + instead. Regression guard for the granite4.2 CI wedge: the /v1 tests + must be able to turn thinking off (runs 33093698028, 33104419835). + """ + from mellea.core.base import CBlock + from mellea.stdlib.context import ChatContext + + backend = OpenAIBackend( + model_id="granite4.2:3b", + base_url="http://localhost:9999/v1", + api_key="test-key", + ) + + with patch.object( + backend._async_client.chat.completions, "create", new_callable=AsyncMock + ) as mock_create: + mock_create.return_value = ChatCompletion( + id="test", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(role="assistant", content="ok"), + ) + ], + created=0, + model="granite4.2:3b", + object="chat.completion", + ) + mot, _ = await backend.generate_from_chat_context( + CBlock(value="hello"), + ChatContext(), + model_options={ModelOption.THINKING: False}, + ) + await mot.avalue() + + call_kwargs = mock_create.call_args.kwargs + assert call_kwargs["reasoning_effort"] == "none" + assert call_kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is False + + +async def test_thinking_false_omits_reasoning_effort_against_real_openai(): + """THINKING=False must NOT send reasoning_effort="none" to api.openai.com. + + "none" is only accepted by newer OpenAI reasoning models; most current + ones reject it outright. reasoning_effort="none" is a workaround for + Ollama's /v1 endpoint specifically (see + test_thinking_false_sends_reasoning_effort_none_and_disable) and must not + reach a real OpenAI target, where it would turn previously-working + THINKING=False calls into a 400. chat_template_kwargs.enable_thinking is + still set for vLLM-style servers reached through this same code path. + """ + from mellea.core.base import CBlock + from mellea.stdlib.context import ChatContext + + backend = OpenAIBackend( + model_id="o3", base_url="https://api.openai.com/v1", api_key="test-key" + ) + + with patch.object( + backend._async_client.chat.completions, "create", new_callable=AsyncMock + ) as mock_create: + mock_create.return_value = ChatCompletion( + id="test", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(role="assistant", content="ok"), + ) + ], + created=0, + model="o3", + object="chat.completion", + ) + mot, _ = await backend.generate_from_chat_context( + CBlock(value="hello"), + ChatContext(), + model_options={ModelOption.THINKING: False}, + ) + await mot.avalue() + + call_kwargs = mock_create.call_args.kwargs + assert "reasoning_effort" not in call_kwargs + assert call_kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is False + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/test/backends/test_openai_vllm.py b/test/backends/test_openai_vllm.py index 70baf73876..30576b866b 100644 --- a/test/backends/test_openai_vllm.py +++ b/test/backends/test_openai_vllm.py @@ -4,6 +4,7 @@ import os import signal import subprocess +import sys import time import openai @@ -31,12 +32,15 @@ import mellea.backends.model_ids as model_ids from mellea import MelleaSession from mellea.backends import ModelOption -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B from mellea.backends.openai import OpenAIBackend from mellea.core import CBlock, ModelOutputThunk from mellea.formatters import TemplateFormatter from mellea.stdlib.context import ChatContext +assert IBM_GRANITE_4_2_3B.hf_model_name is not None +_VLLM_MODEL = os.environ.get("VLLM_TEST_MODEL", IBM_GRANITE_4_2_3B.hf_model_name) + @pytest.fixture(scope="module") def vllm_process(): @@ -61,8 +65,24 @@ def vllm_process(): # bootstrapped here for direct pytest invocations. vllm_venv = os.environ.get("VLLM_VENV_PATH", ".vllm-venv") vllm_python = os.path.join(vllm_venv, "bin", "python") - if not os.path.isfile(vllm_python): - subprocess.run(["uv", "venv", vllm_venv, "--python", "3.11"], check=True) + expected_python = f"{sys.version_info.major}.{sys.version_info.minor}" + existing_python = ( + subprocess.run( + [ + vllm_python, + "-c", + "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ], + capture_output=True, + text=True, + ) + if os.path.isfile(vllm_python) + else None + ) + if existing_python is None or existing_python.stdout.strip() != expected_python: + subprocess.run( + ["uv", "venv", vllm_venv, "--python", sys.executable, "--clear"], check=True + ) subprocess.run( ["uv", "pip", "install", "--python", vllm_python, "vllm"], check=True ) @@ -75,9 +95,9 @@ def vllm_process(): "-m", "vllm.entrypoints.openai.api_server", "--model", - IBM_GRANITE_4_1_3B.hf_model_name, + _VLLM_MODEL, "--served-model-name", - IBM_GRANITE_4_1_3B.hf_model_name, + _VLLM_MODEL, "--enable-lora", "--dtype", "bfloat16", @@ -169,8 +189,8 @@ def backend(gh_run: int, vllm_process: subprocess.Popen): """Shared OpenAI backend configured for vLLM.""" base_url = os.environ.get("VLLM_TEST_BASE_URL", "http://127.0.0.1:8000") + "/v1" return OpenAIBackend( - model_id=IBM_GRANITE_4_1_3B.hf_model_name, # type: ignore - formatter=TemplateFormatter(model_id=IBM_GRANITE_4_1_3B.hf_model_name), # type: ignore + model_id=_VLLM_MODEL, + formatter=TemplateFormatter(model_id=_VLLM_MODEL), base_url=base_url, api_key="EMPTY", ) diff --git a/test/backends/test_tool_calls.py b/test/backends/test_tool_calls.py index 05aef52a01..2b21ce03b3 100644 --- a/test/backends/test_tool_calls.py +++ b/test/backends/test_tool_calls.py @@ -4,6 +4,7 @@ import pytest from mellea.backends import ModelOption +from mellea.backends.model_ids import IBM_GRANITE_4_HYBRID_MICRO from mellea.backends.ollama import OllamaModelBackend from mellea.backends.tools import ( AbstractMelleaTool, @@ -23,7 +24,14 @@ @pytest.fixture(scope="module") def m() -> MelleaSession: - return MelleaSession(backend=OllamaModelBackend(), ctx=ChatContext()) + assert IBM_GRANITE_4_HYBRID_MICRO.ollama_name is not None + return MelleaSession( + backend=OllamaModelBackend( + model_id=IBM_GRANITE_4_HYBRID_MICRO, + model_options={ModelOption.CONTEXT_WINDOW: 2048}, + ), + ctx=ChatContext(), + ) @pytest.fixture(scope="module") diff --git a/test/backends/test_vision_ollama.py b/test/backends/test_vision_ollama.py index c8042afe3c..1993da0322 100644 --- a/test/backends/test_vision_ollama.py +++ b/test/backends/test_vision_ollama.py @@ -110,7 +110,7 @@ def test_image_block_construction_from_pil(pil_image: Image.Image): @pytest.fixture def mocked_session(mock_ollama_backend): canned = ollama.ChatResponse( - model="granite4.1:3b", + model="granite4.2:3b", created_at=None, message=ollama.Message(role="assistant", content="no"), done=True, diff --git a/test/backends/test_vision_openai.py b/test/backends/test_vision_openai.py index 755852bf12..ec18a39140 100644 --- a/test/backends/test_vision_openai.py +++ b/test/backends/test_vision_openai.py @@ -14,29 +14,27 @@ from mellea import MelleaSession, start_session from mellea.backends import ModelOption -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea.backends.model_ids import IBM_GRANITE_VISION_4_1_4B from mellea.core import ImageBlock, ImageUrlBlock, ModelOutputThunk from mellea.stdlib.components import Instruction, Message +assert IBM_GRANITE_VISION_4_1_4B.ollama_name is not None +VISION_MODEL = IBM_GRANITE_VISION_4_1_4B.ollama_name +VISION_CONTEXT_WINDOW = 4096 + @pytest.fixture(scope="module") -def m_session(gh_run): - if gh_run == 1: - m = start_session( - "openai", - model_id=IBM_GRANITE_4_1_3B.ollama_name, # type: ignore - base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", - api_key="ollama", - model_options={ModelOption.MAX_NEW_TOKENS: 5}, - ) - else: - m = start_session( - "openai", - model_id="granite3.2-vision", - base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", - api_key="ollama", - model_options={ModelOption.MAX_NEW_TOKENS: 5}, - ) +def m_session(): + m = start_session( + "openai", + model_id=VISION_MODEL, + base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", + api_key="ollama", + model_options={ + ModelOption.MAX_NEW_TOKENS: 5, + ModelOption.CONTEXT_WINDOW: VISION_CONTEXT_WINDOW, + }, + ) yield m del m diff --git a/test/conftest.py b/test/conftest.py index 43ad43b8be..25bc35a7ca 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,13 +6,12 @@ import os import subprocess import sys -from urllib.parse import urlsplit import pytest import requests from mellea.core import MelleaLogger -from test._ollama_utils import evict_all_loaded_ollama_models +from test._ollama_utils import evict_all_loaded_ollama_models, resolve_ollama_base_url # ============================================================================ # HuggingFace Hub Skip Helper @@ -63,7 +62,8 @@ def _check_ollama_available(): """Check if Ollama is available by checking if port 11434 is listening. Note: This only checks if Ollama is running, not which models are loaded. - Tests may still fail if required models (e.g., granite4.1:3b) are not pulled. + Tests may still fail if required models (e.g., granite4.2:3b) are + not pulled. """ import socket @@ -204,7 +204,7 @@ def gh_run() -> int: } # Execution order when --group-by-backend is used -BACKEND_GROUP_ORDER = ["huggingface", "openai_vllm", "ollama", "api"] +BACKEND_GROUP_ORDER = ["huggingface", "ollama", "openai_vllm", "api"] # ============================================================================ @@ -473,6 +473,16 @@ def cleanup_gpu_backend(backend, backend_name="unknown"): # Test Collection Filtering # ============================================================================ +# Transient Ollama-timeout shapes that the flaky retry net covers (matched +# against `f"{excinfo.type.__name__}: {excinfo.value}"`): +# - "ReadTimeout" native OllamaModelBackend (httpx.ReadTimeout) +# - "APITimeoutError" LiteLLM /v1 path (litellm.Timeout message) +# - "TimeoutError" streaming stream-guard abort (stalled-chunk timeout) +# Deliberately NOT matched: pytest-timeout's watchdog kill — it already spent +# the attempt's whole time budget, so retrying it just burns the next one. +# See test/test_flaky_ollama_rerun.py for the pinned match behaviour. +OLLAMA_TIMEOUT_RERUN_PATTERNS = ["ReadTimeout", "APITimeoutError", "TimeoutError"] + def pytest_collection_modifyitems(config, items): """Skip tests at collection time based on markers and optionally reorder by backend. @@ -485,6 +495,7 @@ def pytest_collection_modifyitems(config, items): skip_ollama = pytest.mark.skip( reason="Ollama not available (port 11434 not listening)" ) + skip_vllm = pytest.mark.skip(reason="vLLM disabled by WITH_VLLM=0") # Auto-apply 'unit' marker to tests without explicit granularity markers. # This enables `pytest -m unit` without per-file maintenance burden. @@ -497,13 +508,19 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip_ollama) else: # Ollama stalls a request past its read timeout on loaded - # runners; retry only that transient error, not real failures. + # runners; retry only those transient timeout shapes (see + # OLLAMA_TIMEOUT_RERUN_PATTERNS), not real failures. item.add_marker( pytest.mark.flaky( - reruns=2, reruns_delay=5, only_rerun="ReadTimeout" + reruns=2, + reruns_delay=5, + only_rerun=OLLAMA_TIMEOUT_RERUN_PATTERNS, ) ) + if os.environ.get("WITH_VLLM") == "0" and item.get_closest_marker("vllm"): + item.add_marker(skip_vllm) + # Auto-apply unit marker if not any(item.get_closest_marker(m) for m in _NON_UNIT): item.add_marker(pytest.mark.unit) @@ -563,7 +580,8 @@ def pytest_runtest_setup(item): # Track backend group transitions when --group-by-backend is used if config.getoption("--group-by-backend", default=False): current_group = None - for group_name, group_info in BACKEND_GROUPS.items(): + for group_name in BACKEND_GROUP_ORDER: + group_info = BACKEND_GROUPS[group_name] markers = group_info.get("markers") or [group_info["marker"]] if any(item.get_closest_marker(m) for m in markers): current_group = group_name @@ -583,29 +601,26 @@ def pytest_runtest_setup(item): # Warm up Ollama models when entering Ollama group if current_group == "ollama" and prev_group != "ollama": logger = MelleaLogger.get_logger() - host_str = os.environ.get("OLLAMA_HOST", "127.0.0.1:11434") - parsed_host_str = urlsplit(host_str) - if parsed_host_str.port: - ollama_base = ( - f"http://{host_str}" if not parsed_host_str.scheme else host_str - ) - else: - port = os.environ.get("OLLAMA_PORT", "11434") - ollama_base = ( - f"http://{host_str}:{port}" - if not parsed_host_str.scheme - else host_str - ) + ollama_base = resolve_ollama_base_url() logger.info( "Warming up ollama models before ollama group (keep_alive=-1)..." ) - for model in ["granite4.1:3b", "granite3.2-vision"]: + # Warm each model at the num_ctx its live tests actually use: a + # later request needing a bigger context forces a runner reload, + # and Ollama 0.33.1 (x86) can ignore num_predict after one. + warmup_models = { + "granite4.2:3b": 8192, + "granite4:micro-h": 2048, + "hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M": 4096, + } + for model, num_ctx in warmup_models.items(): try: requests.post( f"{ollama_base}/api/generate", json={ "model": model, "prompt": "hi", + "options": {"num_ctx": num_ctx, "num_predict": 1}, "stream": False, "keep_alive": -1, }, @@ -618,14 +633,13 @@ def pytest_runtest_setup(item): # Evict Ollama models when leaving Ollama group if prev_group == "ollama" and current_group != "ollama": logger = MelleaLogger.get_logger() - host_str = os.environ.get("OLLAMA_HOST", "127.0.0.1:11434") - if ":" in host_str: - ollama_base = f"http://{host_str}" - else: - port = os.environ.get("OLLAMA_PORT", "11434") - ollama_base = f"http://{host_str}:{port}" + ollama_base = resolve_ollama_base_url() logger.info("Evicting ollama models from VRAM after ollama group...") - for model in ["granite4.1:3b", "granite3.2-vision"]: + for model in [ + "granite4.2:3b", + "granite4:micro-h", + "hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M", + ]: try: requests.post( f"{ollama_base}/api/generate", diff --git a/test/core/test_component_typing.py b/test/core/test_component_typing.py index 7f3593c8dd..7549b8df47 100644 --- a/test/core/test_component_typing.py +++ b/test/core/test_component_typing.py @@ -10,7 +10,8 @@ import mellea.stdlib.functional as mfuncs from mellea import MelleaSession, start_session -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B +from mellea.backends.model_options import ModelOption from mellea.backends.ollama import OllamaModelBackend from mellea.core import ( CBlock, @@ -67,14 +68,13 @@ def _parse(self, computed: ModelOutputThunk) -> int: @pytest.fixture(scope="module") -def backend(gh_run: int): +def backend(): """Shared backend.""" - if gh_run == 1: - return OllamaModelBackend( - model_id=IBM_GRANITE_4_1_3B.ollama_name # type: ignore - ) - else: - return OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore + assert IBM_GRANITE_4_2_3B.ollama_name is not None + return OllamaModelBackend( + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ModelOption.THINKING: False}, + ) @pytest.fixture(scope="module") @@ -124,7 +124,7 @@ def test_incorrect_type_override(): # Marking as qualitative for now since there's so much generation required for this. -# Uses granite4.1:3b (3B hybrid, lightweight) in local mode +# Uses granite4.2:3b (3B, lightweight) in local mode @pytest.mark.qualitative @pytest.mark.ollama @pytest.mark.e2e @@ -187,7 +187,6 @@ def test_message_typing(session): @pytest.mark.e2e async def test_generating_with_sampling(session): m = session - m = start_session() class CustomSamplingStrat(BaseSamplingStrategy): @staticmethod diff --git a/test/formatters/test_template_formatter.py b/test/formatters/test_template_formatter.py index 05a32c35db..0a58d36531 100644 --- a/test/formatters/test_template_formatter.py +++ b/test/formatters/test_template_formatter.py @@ -10,7 +10,7 @@ import jinja2.sandbox import pytest -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B, ModelIdentifier +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B, ModelIdentifier from mellea.core import ( CBlock, Component, @@ -302,7 +302,7 @@ def _parse(self, computed: ModelOutputThunk) -> str: def test_load_with_model_id(instr: Instruction): - tf = TemplateFormatter(IBM_GRANITE_4_1_3B) + tf = TemplateFormatter(IBM_GRANITE_4_2_3B) tmpl = tf._load_template(instr.format_for_llm()) assert tmpl.name is not None assert "granite" in tmpl.name, ( diff --git a/test/scripts/run_tests_with_ollama_and_vllm.sh b/test/scripts/run_tests_with_ollama_and_vllm.sh index b5a8c1e616..34965e7e5c 100755 --- a/test/scripts/run_tests_with_ollama_and_vllm.sh +++ b/test/scripts/run_tests_with_ollama_and_vllm.sh @@ -18,7 +18,7 @@ # SKIP_WARMUP=1 ./run_tests_with_ollama_and_vllm.sh # skip ollama model warmup # WITH_EXAMPLES=1 ./run_tests_with_ollama_and_vllm.sh # include docs/examples/ # WITH_TOOLING_TESTS=1 ./run_tests_with_ollama_and_vllm.sh # include test/tooling/ -# WITH_VLLM=1 VLLM_MODEL=ibm-granite/granite-3.3-8b-instruct \ +# WITH_VLLM=1 VLLM_MODEL=ibm-granite/granite-4.2-3b \ # ./run_tests_with_ollama_and_vllm.sh --group-by-backend -v -s # # LSF example: @@ -42,9 +42,11 @@ else OLLAMA_DIR="$HOME/.ollama" fi OLLAMA_BIN="${OLLAMA_BIN:-$(command -v ollama 2>/dev/null || echo "$HOME/.local/bin/ollama")}" +OLLAMA_CONTEXT_LENGTH="${OLLAMA_CONTEXT_LENGTH:-2048}" OLLAMA_MODEL_LIST=( - "granite4.1:3b" - "granite3.2-vision" + "granite4.2:3b" + "granite4:micro-h" + "hf.co/ibm-granite/granite-vision-4.1-4b-GGUF:Q4_K_M" "llama3.2" "qwen2.5vl:7b" ) @@ -61,7 +63,7 @@ if [[ -z "${WITH_VLLM:-}" ]]; then fi fi VLLM_PORT="${VLLM_PORT:-8100}" -VLLM_MODEL="${VLLM_MODEL:-ibm-granite/granite-4.1-3b}" +VLLM_MODEL="${VLLM_MODEL:-ibm-granite/granite-4.2-3b}" VLLM_GPU_MEM="${VLLM_GPU_MEM:-0.4}" VLLM_MAX_MODEL_LEN="${VLLM_MAX_MODEL_LEN:-4096}" VLLM_MAX_NUM_SEQS="${VLLM_MAX_NUM_SEQS:-256}" @@ -107,18 +109,19 @@ cleanup() { } trap cleanup EXIT -# --- Install ollama binary if missing --- -if [[ ! -x "$OLLAMA_BIN" ]]; then - log "Ollama binary not found at $OLLAMA_BIN — downloading latest release..." +# --- Install a compatible Ollama binary --- +OLLAMA_MIN_VERSION="${OLLAMA_MIN_VERSION:-0.32.2}" +ollama_current_version="" +if [[ -x "$OLLAMA_BIN" ]]; then + ollama_current_version=$("$OLLAMA_BIN" --version 2>/dev/null | awk '{print $NF}' | sed 's/^v//') +fi + +if [[ ! -x "$OLLAMA_BIN" ]] || ! printf '%s\n%s\n' "$OLLAMA_MIN_VERSION" "$ollama_current_version" | sort -V -C; then + log "Installing Ollama $OLLAMA_MIN_VERSION (current: ${ollama_current_version:-missing})..." OLLAMA_INSTALL_DIR="$(dirname "$OLLAMA_BIN")" mkdir -p "$OLLAMA_INSTALL_DIR" - # Get latest release tag from GitHub API - OLLAMA_VERSION=$(curl -fsSL https://api.github.com/repos/ollama/ollama/releases/latest \ - | grep '"tag_name"' | head -1 | cut -d'"' -f4) - log "Latest ollama version: $OLLAMA_VERSION" - - DOWNLOAD_URL="https://github.com/ollama/ollama/releases/download/${OLLAMA_VERSION}/ollama-linux-amd64.tar.zst" + DOWNLOAD_URL="https://github.com/ollama/ollama/releases/download/v${OLLAMA_MIN_VERSION}/ollama-linux-amd64.tar.zst" log "Downloading from $DOWNLOAD_URL (includes CUDA libs, ~1.9GB)..." # Extract everything (bin/ollama + lib/ollama/cuda_v*/) into OLLAMA_INSTALL_DIR's parent @@ -127,7 +130,7 @@ if [[ ! -x "$OLLAMA_BIN" ]]; then OLLAMA_PREFIX="$(dirname "$OLLAMA_INSTALL_DIR")" curl -fsSL "$DOWNLOAD_URL" | tar --use-compress-program=unzstd -x -C "$OLLAMA_PREFIX" chmod +x "$OLLAMA_BIN" - log "Installed ollama $OLLAMA_VERSION to $OLLAMA_PREFIX (bin + CUDA libs)" + log "Installed Ollama $OLLAMA_MIN_VERSION to $OLLAMA_PREFIX (bin + CUDA libs)" fi # --- Check if ollama is already running --- @@ -146,6 +149,7 @@ else log "Starting ollama server on ${OLLAMA_HOST}:${OLLAMA_PORT}..." export OLLAMA_HOST="${OLLAMA_HOST}:${OLLAMA_PORT}" export OLLAMA_MODELS="${OLLAMA_DIR}/models" + export OLLAMA_CONTEXT_LENGTH mkdir -p "$OLLAMA_MODELS" # Ensure ollama can find system CUDA libraries @@ -154,6 +158,7 @@ else log "Added system CUDA to LD_LIBRARY_PATH" fi + log "Using Ollama default context length: $OLLAMA_CONTEXT_LENGTH" "$OLLAMA_BIN" serve > "$LOGDIR/ollama.log" 2>&1 & OLLAMA_PID=$! log "Ollama server PID: $OLLAMA_PID" @@ -225,7 +230,7 @@ if [[ "$WITH_VLLM" == "1" ]]; then log "Reusing existing vLLM venv at $VLLM_VENV (KEEP_VLLM_VENV=1)" else log "Creating isolated vLLM venv at $VLLM_VENV ..." - uv venv "$VLLM_VENV" --python 3.11 --clear + uv venv "$VLLM_VENV" --python 3.12 --clear log "Installing vllm into $VLLM_VENV ..." uv pip install --python "$VLLM_VENV/bin/python" vllm \ > "$LOGDIR/vllm_install.log" 2>&1 \ @@ -281,16 +286,22 @@ else fi # WITH_TOOLING_TESTS=1 includes test/tooling/ (ignored by default) -IGNORE_TOOLING="" +PYTEST_ARGS=() if [[ "${WITH_TOOLING_TESTS:-0}" != "1" ]]; then - IGNORE_TOOLING="--ignore=tooling" + PYTEST_ARGS+=("--ignore=tooling") log "Tooling tests disabled (WITH_TOOLING_TESTS=0). Pass WITH_TOOLING_TESTS=1 to include test/tooling/." fi +if [[ "$#" -eq 0 ]]; then + PYTEST_ARGS+=("--group-by-backend") +else + PYTEST_ARGS+=("$@") +fi + # --- Run tests --- log "Starting pytest..." log "Log directory: $LOGDIR" -log "Pytest args: ${*---group-by-backend}" +log "Pytest args: ${PYTEST_ARGS[*]}" ${UV_PYTHON:+log "Python version: $UV_PYTHON"} # Use UV_PYTHON env var if set, otherwise use default Python @@ -305,11 +316,11 @@ uv run --quiet --frozen --all-groups --all-extras $UV_PYTHON_ARG \ python -c "import nltk; nltk.download('punkt_tab', quiet=True)" || true uv run --quiet --frozen --all-groups --all-extras $UV_PYTHON_ARG \ - pytest "$PYTEST_DIR" $IGNORE_TOOLING ${@---group-by-backend} \ + pytest "$PYTEST_DIR" "${PYTEST_ARGS[@]}" \ 2>&1 | tee "$LOGDIR/pytest_full.log" EXIT_CODE=${PIPESTATUS[0]} log "Tests finished with exit code: $EXIT_CODE" log "Logs: $LOGDIR/" -exit $EXIT_CODE \ No newline at end of file +exit $EXIT_CODE diff --git a/test/stdlib/components/test_genstub.py b/test/stdlib/components/test_genstub.py index b134495143..d787ec56cb 100644 --- a/test/stdlib/components/test_genstub.py +++ b/test/stdlib/components/test_genstub.py @@ -2,12 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +from collections.abc import Iterator from typing import Literal import pytest -from mellea import MelleaSession, generative, start_session -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea import MelleaSession, generative +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B +from mellea.backends.model_options import ModelOption from mellea.backends.ollama import OllamaModelBackend from mellea.core import Requirement from mellea.stdlib.components.genstub import ( @@ -20,19 +22,18 @@ from mellea.stdlib.requirements import simple_validate from mellea.stdlib.sampling import RejectionSamplingStrategy -# Module-level markers: Uses granite4.1:3b (3B, lightweight) in local mode +# Module-level markers: Uses granite4.2:3b (3B, lightweight) in local mode pytestmark = [pytest.mark.ollama, pytest.mark.e2e] @pytest.fixture(scope="module") -def backend(gh_run: int): +def backend(): """Shared backend.""" - if gh_run == 1: - return OllamaModelBackend( - model_id=IBM_GRANITE_4_1_3B.ollama_name # type: ignore - ) - else: - return OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore + assert IBM_GRANITE_4_2_3B.ollama_name is not None + return OllamaModelBackend( + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ModelOption.THINKING: False}, + ) @generative @@ -48,9 +49,9 @@ async def async_write_short_sentence(topic: str) -> str: ... @pytest.fixture(scope="function") -def session(): +def session(backend) -> Iterator[MelleaSession]: """Fresh session for each test.""" - session = start_session() + session = MelleaSession(backend=backend) yield session session.reset() diff --git a/test/stdlib/test_base_context.py b/test/stdlib/test_base_context.py index 915903ef2f..52311d5ed5 100644 --- a/test/stdlib/test_base_context.py +++ b/test/stdlib/test_base_context.py @@ -419,12 +419,12 @@ def test_session_does_not_bind_when_context_has_history(): def test_get_context_length_litellm_prefixed_string(): - # LiteLLM prefixes model names with a provider slug, e.g. "ollama_chat/granite4.1:3b". + # LiteLLM prefixes model names with a provider slug, e.g. "ollama_chat/granite4.2:3b". # The stripped bare name should resolve to the correct context length. - assert get_context_length("ollama_chat/granite4.1:3b") == 131072 - assert get_context_length("ollama/granite4.1:8b") == 131072 + assert get_context_length("ollama_chat/granite4.2:3b") == 131072 + assert get_context_length("ollama/granite4.2:8b") == 131072 # Multi-segment strip still resolves when the remainder is a known HF name. - assert get_context_length("huggingface/ibm-granite/granite-4.1-3b") == 131072 + assert get_context_length("huggingface/ibm-granite/granite-4.2-3b") == 131072 # Completely unknown prefix+name returns None. assert get_context_length("someprefix/not-a-real-model") is None diff --git a/test/stdlib/test_spans.py b/test/stdlib/test_spans.py index 0281abcac9..8dc9091530 100644 --- a/test/stdlib/test_spans.py +++ b/test/stdlib/test_spans.py @@ -9,7 +9,7 @@ "llguidance", reason="llguidance not installed — install mellea[hf]" ) from mellea.backends.huggingface import LocalHFBackend -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B from mellea.core import CBlock from mellea.stdlib.components import SimpleComponent from mellea.stdlib.context import ChatContext @@ -17,7 +17,7 @@ from test.conftest import hf_skip from test.predicates import require_gpu -# Module-level markers for all tests using Granite 4.1 3B model +# Module-level markers for all tests using Granite 4.2 3B model pytestmark = [pytest.mark.huggingface, require_gpu(min_vram_gb=12), pytest.mark.e2e] @@ -27,8 +27,8 @@ def m_session(gh_run): with hf_skip(): m = start_session( "hf", - model_id=IBM_GRANITE_4_1_3B, - model_options={ModelOption.MAX_NEW_TOKENS: 64}, + model_id=IBM_GRANITE_4_2_3B, + model_options={ModelOption.MAX_NEW_TOKENS: 64, ModelOption.THINKING: False}, ) yield m @@ -77,7 +77,7 @@ async def test_kv(m_session) -> None: response = await backend._generate_from_context_with_kv_cache( action=CBlock("What is the street address of the MIT-IBM Watson AI Lab?"), ctx=ctx, - model_options=dict(), + model_options={ModelOption.THINKING: False}, ) result = await response.avalue() assert "314" in result, f"Expected correct answer (314 main st) but found: {result}" diff --git a/test/telemetry/test_metrics_backend.py b/test/telemetry/test_metrics_backend.py index 1643b4f064..d34ff2adaf 100644 --- a/test/telemetry/test_metrics_backend.py +++ b/test/telemetry/test_metrics_backend.py @@ -6,11 +6,12 @@ Tests that backends correctly record token metrics through the telemetry system. """ +import asyncio import os import pytest -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B, IBM_GRANITE_4_HYBRID_SMALL +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B, IBM_GRANITE_4_HYBRID_SMALL from mellea.plugins.manager import ( disable_background_collection, discard_background_tasks, @@ -18,7 +19,7 @@ enable_background_collection, ) from mellea.stdlib.components import Message -from mellea.stdlib.context import SimpleContext +from mellea.stdlib.context import ChatContext from test.conftest import hf_skip from test.predicates import require_api_key, require_gpu from test.telemetry.conftest import reset_metrics_state @@ -37,6 +38,10 @@ pytest.mark.e2e, ] +# Match granite4.2:3b's constrained default (Modelfile num_ctx: 8192) so the +# runner is loaded once and never reloaded for a context-size mismatch. +TEST_CONTEXT_WINDOW = 8192 + @pytest.fixture def metric_reader(): @@ -73,7 +78,7 @@ def hf_metrics_backend(gh_run): with hf_skip(): backend = LocalHFBackend( - model_id=IBM_GRANITE_4_1_3B.hf_model_name, # type: ignore + model_id=IBM_GRANITE_4_2_3B.hf_model_name, # type: ignore cache=SimpleLRUCache(5), ) @@ -173,9 +178,25 @@ async def test_ollama_token_metrics_integration( monkeypatch.setenv("MELLEA_GENERATION_CHUNK_EVENTS", "true") provider = _setup_metrics_provider(metrics_module, metric_reader) - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext() - ctx = ctx.add(Message(role="user", content="Say 'hello' and nothing else")) + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + # Bound the worst case: open-ended prompts have produced + # 1800+ token generations on CI, and at CI's ~9 t/s CPU decode + # that is a 3-15 minute single request that can eat the test's + # entire pytest-timeout budget. 64 tokens is far more than a + # counting answer needs. + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext() + # A counting prompt reliably spans many output tokens, so the streaming + # branch always sees >=2 chunks (the time_per_output_chunk histogram only + # records inter-chunk intervals; a single-chunk reply like "Hello!" leaves + # it empty — observed in run 33163851256). + ctx = ctx.add(Message(role="user", content="Count from 1 to 10 and nothing else")) model_options = {ModelOption.STREAM: True} if stream else {} mot, _ = await backend.generate_from_context( @@ -184,8 +205,19 @@ async def test_ollama_token_metrics_integration( # For streaming, consume the stream fully before checking metrics if stream: - await mot.astream() - await mot.avalue() + # The 120 s per-chunk stream guard does not bound total request time: + # a stalled or queued stream that keeps the HTTP connection alive + # (observed on CI: a 900 s /v1 stream, run 33015176815) keeps the + # per-chunk guard re-armed and rides to the 900 s pytest watchdog, + # killing the job. Bound the stream to the same 300 s the + # non-streaming paths use so a genuine stall surfaces as a bounded + # TimeoutError the flaky marker retries, matching non-streaming + # behaviour. + await asyncio.wait_for(mot.astream(), timeout=300.0) + # astream() returns as soon as its queue drains, so a long stream is + # finished by avalue(); keep that inside the same 300 s budget (run + # 33048969379: a 15 m stream escaped the astream bound above). + await asyncio.wait_for(mot.avalue(), timeout=300.0) # Force metrics export and collection await drain_background_tasks() @@ -248,11 +280,33 @@ async def test_openai_token_metrics_integration(enable_metrics, metric_reader, s # Use Ollama's OpenAI-compatible endpoint backend = OpenAIBackend( - model_id=IBM_GRANITE_4_1_3B.ollama_name, # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, # type: ignore base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", api_key="ollama", + # granite4.2 thinks by default and Ollama's /v1 endpoint exposes no + # template-kwargs control for it: THINKING: False sends + # reasoning_effort="none", which the CI-pinned Ollama 0.33.1 maps to + # think=false. Without it a 64-token generation is ~45 thinking + # tokens — minutes on a 4-vCPU runner, blowing the 300 s request cap + # whenever the runner is 2-3x slower than nominal (runs + # 33093698028, 33104419835). The vLLM-style + # chat_template_kwargs.enable_thinking workaround was ignored by + # Ollama, which is why every /v1 CI run wedged in the openai/litellm + # phase. + # Same output bound as the other live "say hello" tests: without it a + # non-compliant generation (observed 1800+ tokens on CI) can run for + # minutes on a ~9 t/s CPU runner and eat the test's entire budget. + model_options={ModelOption.THINKING: False, ModelOption.MAX_NEW_TOKENS: 64}, + # Disable the OpenAI SDK's automatic retries and bound each request to + # the 300 s the other live paths use. With the SDK defaults (2 retries, + # 600 s read timeout) a stalled server multiplies into a + # 600 s + 600 s chain inside one attempt (observed in run + # 33039206380: a request timed out at 10m0s, the SDK retried, and the + # second attempt was killed by the 900 s pytest watchdog mid-flight). + max_retries=0, + timeout=300.0, ) - ctx = SimpleContext() + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'hello' and nothing else")) model_options = {ModelOption.STREAM: True} if stream else {} @@ -262,8 +316,19 @@ async def test_openai_token_metrics_integration(enable_metrics, metric_reader, s # For streaming, consume the stream fully before checking metrics if stream: - await mot.astream() - await mot.avalue() + # The 120 s per-chunk stream guard does not bound total request time: + # a stalled or queued stream that keeps the HTTP connection alive + # (observed on CI: a 900 s /v1 stream, run 33015176815) keeps the + # per-chunk guard re-armed and rides to the 900 s pytest watchdog, + # killing the job. Bound the stream to the same 300 s the + # non-streaming paths use so a genuine stall surfaces as a bounded + # TimeoutError the flaky marker retries, matching non-streaming + # behaviour. + await asyncio.wait_for(mot.astream(), timeout=300.0) + # astream() returns as soon as its queue drains, so a long stream is + # finished by avalue(); keep that inside the same 300 s budget (run + # 33048969379: a 15 m stream escaped the astream bound above). + await asyncio.wait_for(mot.avalue(), timeout=300.0) await drain_background_tasks() provider.force_flush() @@ -311,7 +376,7 @@ async def test_watsonx_token_metrics_integration(enable_metrics, metric_reader): model_id=IBM_GRANITE_4_HYBRID_SMALL.watsonx_name, # type: ignore project_id=os.getenv("WATSONX_PROJECT_ID", "test-project"), ) - ctx = SimpleContext() + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'hello' and nothing else")) mot, _ = await backend.generate_from_context( @@ -370,9 +435,33 @@ async def test_litellm_token_metrics_integration( provider = _setup_metrics_provider(metrics_module, metric_reader) # Use LiteLLM with openai/ prefix - it will use the OPENAI_BASE_URL env var - # This tests LiteLLM with a provider that properly returns token usage - backend = LiteLLMBackend(model_id=f"openai/{IBM_GRANITE_4_1_3B.ollama_name}") # type: ignore - ctx = SimpleContext() + # This tests LiteLLM with a provider that properly returns token usage. + # + # "timeout" bounds each attempt to the same 300 s the native + # OllamaModelBackend uses, and "num_retries": 0 stops the OpenAI SDK from + # silently retrying a stalled attempt: without both, a single stalled + # request can consume the whole 900 s pytest-timeout budget (litellm falls + # back to a 600 s per-attempt timeout and the SDK retries twice), so the + # test dies to the watchdog with no retryable error. With the bound in + # place, a stall raises litellm.Timeout, whose APITimeoutError message the + # conftest flaky marker (OLLAMA_TIMEOUT_RERUN_PATTERNS) retries like a + # native ReadTimeout. + backend = LiteLLMBackend( # type: ignore + model_id=f"openai/{IBM_GRANITE_4_2_3B.ollama_name}", + # MAX_NEW_TOKENS: same output bound as the other live "say hello" + # tests (see test_ollama_token_metrics_integration). THINKING: False + # is required: granite4.2 thinks by default and a 64-token + # generation is ~45 thinking tokens — minutes on a 4-vCPU CI runner, + # blowing the 300 s request cap (runs 33093698028, 33104419835); + # mellea maps it to reasoning_effort="none" for the /v1 endpoint. + model_options={ + ModelOption.THINKING: False, + "timeout": 300.0, + "num_retries": 0, + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'hello' and nothing else")) model_options = {ModelOption.STREAM: True} if stream else {} @@ -382,8 +471,19 @@ async def test_litellm_token_metrics_integration( # For streaming, consume the stream fully before checking metrics if stream: - await mot.astream() - await mot.avalue() + # The 120 s per-chunk stream guard does not bound total request time: + # a stalled or queued stream that keeps the HTTP connection alive + # (observed on CI: a 900 s /v1 stream, run 33015176815) keeps the + # per-chunk guard re-armed and rides to the 900 s pytest watchdog, + # killing the job. Bound the stream to the same 300 s the + # non-streaming paths use so a genuine stall surfaces as a bounded + # TimeoutError the flaky marker retries, matching non-streaming + # behaviour. + await asyncio.wait_for(mot.astream(), timeout=300.0) + # astream() returns as soon as its queue drains, so a long stream is + # finished by avalue(); keep that inside the same 300 s budget (run + # 33048969379: a 15 m stream escaped the astream bound above). + await asyncio.wait_for(mot.avalue(), timeout=300.0) await drain_background_tasks() provider.force_flush() @@ -430,7 +530,7 @@ async def test_huggingface_token_metrics_integration( provider = _setup_metrics_provider(metrics_module, metric_reader) - ctx = SimpleContext() + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'hello' and nothing else")) model_options = {ModelOption.STREAM: True} if stream else {} @@ -440,8 +540,19 @@ async def test_huggingface_token_metrics_integration( # For streaming, consume the stream fully before checking metrics if stream: - await mot.astream() - await mot.avalue() + # The 120 s per-chunk stream guard does not bound total request time: + # a stalled or queued stream that keeps the HTTP connection alive + # (observed on CI: a 900 s /v1 stream, run 33015176815) keeps the + # per-chunk guard re-armed and rides to the 900 s pytest watchdog, + # killing the job. Bound the stream to the same 300 s the + # non-streaming paths use so a genuine stall surfaces as a bounded + # TimeoutError the flaky marker retries, matching non-streaming + # behaviour. + await asyncio.wait_for(mot.astream(), timeout=300.0) + # astream() returns as soon as its queue drains, so a long stream is + # finished by avalue(); keep that inside the same 300 s budget (run + # 33048969379: a 15 m stream escaped the astream bound above). + await asyncio.wait_for(mot.avalue(), timeout=300.0) await drain_background_tasks() provider.force_flush() @@ -481,7 +592,7 @@ async def test_error_metrics_on_backend_failure(enable_metrics, metric_reader): base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", api_key="dummy", ) - ctx = SimpleContext() + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say hello")) mot, _ = await backend.generate_from_context( @@ -514,17 +625,22 @@ async def test_error_metrics_on_backend_failure(enable_metrics, metric_reader): @pytest.mark.ollama async def test_ollama_sampling_metrics_integration(enable_metrics, metric_reader): """Test that sampling metrics are recorded through a full RejectionSamplingStrategy loop.""" + from mellea.backends.model_options import ModelOption from mellea.backends.ollama import OllamaModelBackend from mellea.stdlib.components import Instruction - from mellea.stdlib.context import SimpleContext + from mellea.stdlib.context import ChatContext from mellea.stdlib.sampling import RejectionSamplingStrategy from mellea.telemetry import metrics as metrics_module provider = _setup_metrics_provider(metrics_module, metric_reader) - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + # Same output bound as the other live "say hello" tests. + model_options={ModelOption.THINKING: False, ModelOption.MAX_NEW_TOKENS: 64}, + ) strategy = RejectionSamplingStrategy(loop_budget=1) - ctx = SimpleContext() + ctx = ChatContext() result = await strategy.sample( action=Instruction("Say hello"), context=ctx, backend=backend, requirements=None @@ -558,15 +674,20 @@ async def test_ollama_generate_from_raw_metrics_integration( enable_metrics, metric_reader ): """Token and latency metrics are recorded for `generate_from_raw` calls.""" + from mellea.backends.model_options import ModelOption from mellea.backends.ollama import OllamaModelBackend from mellea.core import CBlock - from mellea.stdlib.context import SimpleContext + from mellea.stdlib.context import ChatContext from mellea.telemetry import metrics as metrics_module provider = _setup_metrics_provider(metrics_module, metric_reader) - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext() + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + # Same output bound as the other live "say hello" tests. + model_options={ModelOption.THINKING: False, ModelOption.MAX_NEW_TOKENS: 64}, + ) + ctx = ChatContext() actions = [CBlock("Say 'hi' and nothing else."), CBlock("Say 'hello'.")] results = await backend.generate_from_raw(actions, ctx=ctx) @@ -599,7 +720,7 @@ async def test_generate_from_raw_error_metrics_integration( """Test that error metrics are recorded when `generate_from_raw` fails.""" from mellea.backends.openai import OpenAIBackend from mellea.core import CBlock - from mellea.stdlib.context import SimpleContext + from mellea.stdlib.context import ChatContext from mellea.telemetry import metrics as metrics_module provider = _setup_metrics_provider(metrics_module, metric_reader) @@ -609,7 +730,7 @@ async def test_generate_from_raw_error_metrics_integration( base_url=f"http://{os.environ.get('OLLAMA_HOST', 'localhost:11434')}/v1", api_key="dummy", ) - ctx = SimpleContext() + ctx = ChatContext() actions = [CBlock("Say 'hi'.")] with pytest.raises(Exception): diff --git a/test/telemetry/test_tracing_backend.py b/test/telemetry/test_tracing_backend.py index 2e98e710de..9d3cbe8835 100644 --- a/test/telemetry/test_tracing_backend.py +++ b/test/telemetry/test_tracing_backend.py @@ -9,7 +9,8 @@ import ollama import pytest -from mellea.backends.model_ids import IBM_GRANITE_4_1_3B +from mellea.backends.model_ids import IBM_GRANITE_4_2_3B +from mellea.backends.model_options import ModelOption from mellea.backends.ollama import OllamaModelBackend from mellea.plugins.manager import ( disable_background_collection, @@ -18,7 +19,7 @@ enable_background_collection, ) from mellea.stdlib.components import Message -from mellea.stdlib.context import SimpleContext +from mellea.stdlib.context import ChatContext from test.telemetry.conftest import reset_tracing_state # Check if OpenTelemetry is available @@ -37,6 +38,10 @@ not OTEL_AVAILABLE, reason="OpenTelemetry not installed" ) +# Match granite4.2:3b's constrained default (Modelfile num_ctx: 8192) so the +# runner is loaded once and never reloaded for a context-size mismatch. +TEST_CONTEXT_WINDOW = 8192 + @pytest.fixture(scope="module", autouse=True) def setup_telemetry(): @@ -160,7 +165,7 @@ async def fake_chat_stream(*args, **kwargs): mock_async_client_cls.return_value = mock_async_instance backend = OllamaModelBackend(model_id="test-model") - ctx = SimpleContext().add(Message(role="user", content="Count to 3")) + ctx = ChatContext().add(Message(role="user", content="Count to 3")) async with await stream( Message(role="assistant", content=""), backend, ctx @@ -215,9 +220,7 @@ async def test_span_duration_captures_async_operation_mocked( span_exporter, mocked_tracing_backend ): """Test span duration without requiring a live Ollama server.""" - ctx = SimpleContext().add( - Message(role="user", content="Say 'test' and nothing else") - ) + ctx = ChatContext().add(Message(role="user", content="Say 'test' and nothing else")) mot, _ = await mocked_tracing_backend.generate_from_context( Message(role="assistant", content=""), ctx @@ -248,9 +251,7 @@ async def test_context_propagation_parent_child_mocked( span_exporter, mocked_tracing_backend ): """Test parent-child span propagation without a live Ollama server.""" - ctx = SimpleContext().add( - Message(role="user", content="Say 'test' and nothing else") - ) + ctx = ChatContext().add(Message(role="user", content="Say 'test' and nothing else")) from mellea.telemetry import tracing @@ -287,9 +288,7 @@ async def test_token_usage_recorded_after_completion_mocked( span_exporter, mocked_tracing_backend ): """Test deterministic token usage without a live Ollama server.""" - ctx = SimpleContext().add( - Message(role="user", content="Say 'test' and nothing else") - ) + ctx = ChatContext().add(Message(role="user", content="Say 'test' and nothing else")) mot, _ = await mocked_tracing_backend.generate_from_context( Message(role="assistant", content=""), ctx @@ -320,7 +319,7 @@ async def test_span_not_closed_prematurely_mocked( span_exporter, mocked_tracing_backend ): """Test that a mocked async operation keeps its span open until completion.""" - ctx = SimpleContext().add(Message(role="user", content="Count to 5")) + ctx = ChatContext().add(Message(role="user", content="Count to 5")) mot, _ = await mocked_tracing_backend.generate_from_context( Message(role="assistant", content=""), ctx @@ -349,7 +348,7 @@ async def test_multiple_generations_separate_spans_mocked( span_exporter, mocked_tracing_backend ): """Test separate generation spans without a live Ollama server.""" - ctx = SimpleContext().add(Message(role="user", content="Say 'test'")) + ctx = ChatContext().add(Message(role="user", content="Say 'test'")) mot1, _ = await mocked_tracing_backend.generate_from_context( Message(role="assistant", content=""), ctx @@ -381,8 +380,18 @@ async def test_multiple_generations_separate_spans_mocked( async def test_span_duration_captures_async_operation(span_exporter): """Test that span duration includes the full async operation time.""" - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext() + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + # Bound the worst case: without a cap an unexpected long + # generation runs until the client's 300 s request cap (run + # 33152524235: 2542-token runaway on an uncapped request). + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'test' and nothing else")) mot, _ = await backend.generate_from_context( @@ -400,7 +409,7 @@ async def test_span_duration_captures_async_operation(span_exporter): backend_span = None for span in spans: - if span.name == f"chat {IBM_GRANITE_4_1_3B.ollama_name}": + if span.name == f"chat {IBM_GRANITE_4_2_3B.ollama_name}": backend_span = span break @@ -420,8 +429,18 @@ async def test_span_duration_captures_async_operation(span_exporter): async def test_context_propagation_parent_child(span_exporter): """Test that parent-child span relationships are maintained.""" - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext() + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + # Bound the worst case: without a cap an unexpected long + # generation runs until the client's 300 s request cap (run + # 33152524235: 2542-token runaway on an uncapped request). + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'test' and nothing else")) # Create a parent span using the module's own tracer provider @@ -448,7 +467,7 @@ async def test_context_propagation_parent_child(span_exporter): for span in spans: if span.name == "parent_operation": parent_recorded = span - elif span.name == f"chat {IBM_GRANITE_4_1_3B.ollama_name}": # Gen-AI convention + elif span.name == f"chat {IBM_GRANITE_4_2_3B.ollama_name}": # Gen-AI convention child_recorded = span assert parent_recorded is not None, "Parent span not found" @@ -470,8 +489,18 @@ async def test_context_propagation_parent_child(span_exporter): async def test_token_usage_recorded_after_completion(span_exporter): """Test that token usage metrics are recorded after async completion.""" - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext() + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + # Bound the worst case: without a cap an unexpected long + # generation runs until the client's 300 s request cap (run + # 33152524235: 2542-token runaway on an uncapped request). + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'test' and nothing else")) mot, _ = await backend.generate_from_context( @@ -486,9 +515,7 @@ async def test_token_usage_recorded_after_completion(span_exporter): backend_span = None for span in spans: - if ( - span.name == f"chat {IBM_GRANITE_4_1_3B.ollama_name}" - ): # Gen-AI convention uses 'chat {model}' for chat completions + if span.name == f"chat {IBM_GRANITE_4_2_3B.ollama_name}": backend_span = span break @@ -522,8 +549,18 @@ async def test_token_usage_recorded_after_completion(span_exporter): async def test_span_not_closed_prematurely(span_exporter): """Test that spans are not closed before async operations complete.""" - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext() + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + # Bound the worst case: without a cap an unexpected long + # generation runs until the client's 300 s request cap (run + # 33152524235: 2542-token runaway on an uncapped request). + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Count to 5")) mot, _ = await backend.generate_from_context( @@ -534,7 +571,7 @@ async def test_span_not_closed_prematurely(span_exporter): # because we haven't awaited the ModelOutputThunk spans_before = span_exporter.get_finished_spans() backend_spans_before = [ - s for s in spans_before if s.name == f"chat {IBM_GRANITE_4_1_3B.ollama_name}" + s for s in spans_before if s.name == f"chat {IBM_GRANITE_4_2_3B.ollama_name}" ] # Gen-AI convention # Now complete the async operation @@ -544,7 +581,7 @@ async def test_span_not_closed_prematurely(span_exporter): # Now the span should be closed spans_after = span_exporter.get_finished_spans() backend_spans_after = [ - s for s in spans_after if s.name == f"chat {IBM_GRANITE_4_1_3B.ollama_name}" + s for s in spans_after if s.name == f"chat {IBM_GRANITE_4_2_3B.ollama_name}" ] # Gen-AI convention # The span should only appear after completion @@ -559,8 +596,18 @@ async def test_span_not_closed_prematurely(span_exporter): async def test_multiple_generations_separate_spans(span_exporter): """Test that multiple generations create separate spans.""" - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext() + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + # Bound the worst case: without a cap an unexpected long + # generation runs until the client's 300 s request cap (run + # 33152524235: 2542-token runaway on an uncapped request). + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext() ctx = ctx.add(Message(role="user", content="Say 'test'")) # Generate twice @@ -578,7 +625,7 @@ async def test_multiple_generations_separate_spans(span_exporter): # Get the recorded spans spans = span_exporter.get_finished_spans() backend_spans = [ - s for s in spans if s.name == f"chat {IBM_GRANITE_4_1_3B.ollama_name}" + s for s in spans if s.name == f"chat {IBM_GRANITE_4_2_3B.ollama_name}" ] # Gen-AI convention assert len(backend_spans) >= 2, ( @@ -603,8 +650,18 @@ async def test_stream_e2e(span_exporter): """ from mellea.stdlib.streaming import stream - backend = OllamaModelBackend(model_id=IBM_GRANITE_4_1_3B.ollama_name) # type: ignore - ctx = SimpleContext().add(Message(role="user", content="Count to 3")) + backend = OllamaModelBackend( # type: ignore + model_id=IBM_GRANITE_4_2_3B.ollama_name, + model_options={ + ModelOption.CONTEXT_WINDOW: TEST_CONTEXT_WINDOW, + ModelOption.THINKING: False, + # Bound the worst case: without a cap an unexpected long + # generation runs until the client's 300 s request cap (run + # 33152524235: 2542-token runaway on an uncapped request). + ModelOption.MAX_NEW_TOKENS: 64, + }, + ) + ctx = ChatContext().add(Message(role="user", content="Count to 3")) async with await stream( Message(role="assistant", content=""), backend, ctx @@ -618,7 +675,7 @@ async def test_stream_e2e(span_exporter): spans = span_exporter.get_finished_spans() streaming_span = next(s for s in spans if s.name == "stream") chat_span = next( - s for s in spans if s.name == f"chat {IBM_GRANITE_4_1_3B.ollama_name}" + s for s in spans if s.name == f"chat {IBM_GRANITE_4_2_3B.ollama_name}" ) assert streaming_span.parent is None, "streaming span should be a root" diff --git a/test/test_flaky_ollama_rerun.py b/test/test_flaky_ollama_rerun.py new file mode 100644 index 0000000000..83ad7e0477 --- /dev/null +++ b/test/test_flaky_ollama_rerun.py @@ -0,0 +1,161 @@ +# Copyright IBM Corp. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression tests for the Ollama flaky-rerun patterns in `test/conftest.py`. + +The conftest applies `pytest.mark.flaky(only_rerun=OLLAMA_TIMEOUT_RERUN_PATTERNS)` +to every `ollama`-marked test so that transient Ollama stalls are retried while +real failures are not. pytest-rerunfailures matches each pattern with +`re.search` against `f"{excinfo.type.__name__}: {excinfo.value}"`. + +These tests pin that match behaviour against the exception strings each +backend path actually produces, so the retry net cannot silently stop covering +a path (as it did for the LiteLLM path, whose `litellm.exceptions.Timeout` +message the original single `"ReadTimeout"` pattern never matched), and it +cannot start retrying the pytest-timeout watchdog kill, which has already +consumed the whole per-attempt budget. +""" + +import re +from pathlib import Path + +from test.conftest import OLLAMA_TIMEOUT_RERUN_PATTERNS + +# The live "say hello" telemetry tests whose timeout shapes are pinned here. +METRICS_TEST_PATH = Path(__file__).parent / "telemetry" / "test_metrics_backend.py" + +# The quality workflow whose Ollama model provisioning is pinned here. +QUALITY_WORKFLOW_PATH = ( + Path(__file__).parents[1] / ".github" / "workflows" / "quality.yml" +) + +# Match strings captured from the exception shapes each path raises, using the +# same f"{type.__name__}: {value}" construction pytest-rerunfailures uses. + +NATIVE_READ_TIMEOUT = "ReadTimeout: " + +# Captured from litellm 1.95.0 against a stalled OpenAI-compatible endpoint +# (socket accepted, no response) with a bounded request timeout: +LITELLM_TIMEOUT = ( + "Timeout: litellm.Timeout: APITimeoutError - Request timed out. " + "Error_str: Request timed out. - timeout value=300.0, time taken=300.12 seconds" +) + +# The streaming stream-guard abort (mellea/helpers/async_helpers.py, +# DEFAULT_CHUNK_TIMEOUT=120.0): the builtin TimeoutError is raised verbatim at +# the consumer (mellea/core/base.py) when a stalled stream goes quiet for 120 s. +STREAM_GUARD_TIMEOUT = ( + "TimeoutError: Stream timed out after 120.0s without a chunk " + "(covers time-to-first-token and inter-chunk gaps). " + "Set ModelOption.STREAM_TIMEOUT to a larger value or None to disable." +) + +# Raised by asyncio.wait_for when the test-level 300 s total-stream budget +# (test/telemetry/test_metrics_backend.py) is exhausted: no message. +STREAM_WAIT_FOR_TIMEOUT = "TimeoutError: " + +# pytest-timeout watchdog kill of an attempt that consumed the 900 s budget. +WATCHDOG_KILL = "Failed: Timeout (>900.0s) from pytest-timeout." + + +def _matches_any(patterns: list[str], match_string: str) -> bool: + return any( + isinstance(pattern, str) and re.search(pattern, match_string) + for pattern in patterns + ) + + +def test_native_ollama_readtimeout_is_rerunnable(): + """The native OllamaModelBackend timeout (httpx.ReadTimeout) must rerun.""" + assert _matches_any(OLLAMA_TIMEOUT_RERUN_PATTERNS, NATIVE_READ_TIMEOUT) + + +def test_litellm_openai_compatible_timeout_is_rerunnable(): + """The LiteLLM path's litellm.Timeout (APITimeoutError message) must rerun.""" + assert _matches_any(OLLAMA_TIMEOUT_RERUN_PATTERNS, LITELLM_TIMEOUT) + + +def test_streaming_stream_guard_timeout_is_rerunnable(): + """A stalled stream aborted by the 120 s chunk guard must rerun.""" + assert _matches_any(OLLAMA_TIMEOUT_RERUN_PATTERNS, STREAM_GUARD_TIMEOUT) + + +def test_streaming_total_budget_timeout_is_rerunnable(): + """A stalled stream that exhausts the 300 s wait_for budget must rerun. + + asyncio.wait_for raises the builtin TimeoutError with an empty message; + the per-chunk guard only bounds inter-chunk gaps, so a stream that keeps + the connection alive but never finishes needs this total-time backstop. + """ + assert _matches_any(OLLAMA_TIMEOUT_RERUN_PATTERNS, STREAM_WAIT_FOR_TIMEOUT) + + +def test_pytest_timeout_watchdog_kill_is_not_rerunnable(): + """A watchdog kill already spent the attempt budget; rerunning is pure waste.""" + assert not _matches_any(OLLAMA_TIMEOUT_RERUN_PATTERNS, WATCHDOG_KILL) + + +def _function_source(src: str, name: str) -> str: + """Extract one top-level function's source, up to the next top-level def.""" + start = re.search(rf"^(async )?def {re.escape(name)}\(", src, re.MULTILINE) + assert start is not None, f"{name} not found in source" + rest = src[start.end() :] + end = re.search(r"^(async )?def ", rest, re.MULTILINE) + return rest[: end.start()] if end else rest + + +def test_live_openai_compat_tests_bound_output_length(): + """The /v1 live tests must cap output via `ModelOption.MAX_NEW_TOKENS`. + + Without an output cap, a non-compliant generation (observed 1800+ tokens + on CI) can run for minutes on a slow CI runner and eat the test's entire + budget (run 33048969379, 3.12 lane). Both the OpenAI and LiteLLM `/v1` + backends remap `ModelOption.MAX_NEW_TOKENS` to `max_completion_tokens` + on the wire, which Ollama's `/v1` endpoint has honoured since 0.33.1 + (the version now pinned in CI). Unlike the native OllamaModelBackend + tests elsewhere in the same file (which also use MAX_NEW_TOKENS but + never had this bug), these two hit the /v1 remap path. + """ + src = METRICS_TEST_PATH.read_text() + for name in ( + "test_openai_token_metrics_integration", + "test_litellm_token_metrics_integration", + ): + body = _function_source(src, name) + assert "ModelOption.MAX_NEW_TOKENS: 64" in body, ( + f"{name} should bound output via ModelOption.MAX_NEW_TOKENS" + ) + assert '"max_tokens"' not in body, ( + f"{name} should not use the raw max_tokens key — it is remapped " + "to max_completion_tokens on the wire regardless, so the raw " + "key adds no protection and misleads readers into thinking it " + "bypasses the remap" + ) + + +def test_live_tests_bound_the_avalue_consumption(): + """All four live consumption paths must stay inside the 300 s budget. + + astream() returns as soon as its queue drains, so a long stream is + finished by avalue(); without the bound it rides to the 900 s pytest + watchdog (run 33048969379: a 15 m stream escaped the astream bound). + """ + src = METRICS_TEST_PATH.read_text() + assert src.count("asyncio.wait_for(mot.avalue(), timeout=300.0)") == 4 + + +def test_ci_constrains_the_granite_4_2_context_window(): + """CI must re-point granite4.2:3b at a constrained-context build. + + The published granite4.2:3b tag ships `num_ctx 131072` in its Modelfile + (granite4.1:3b carries no num_ctx, so CI ran the 4096 default). On the + 16 GB CPU runner the 131K context allocates a ~6 GB KV cache at load + time (8.35 GB total vs 2.75 GB at 8192, measured locally); the memory + pressure wedges inference for 15-30 minute windows that outlast the + 3x300 s retry budget (runs 33048969379 and 33067783570, tracked in + #1589). main-based runs never hit this, which is why the correlation + with the 4.2 model switch in this PR is exact. + """ + src = QUALITY_WORKFLOW_PATH.read_text() + assert "PARAMETER num_ctx 8192" in src + assert "ollama create granite4.2:3b -f" in src