diff --git a/docs/docs/how-to/use-speech-and-audio.md b/docs/docs/how-to/use-speech-and-audio.md new file mode 100644 index 000000000..07e097e35 --- /dev/null +++ b/docs/docs/how-to/use-speech-and-audio.md @@ -0,0 +1,227 @@ +--- +title: "Use Speech and Audio Input" +description: "Pass audio to instruct() and chat() calls, and check which backends can send it." +sidebar_label: "Use Speech and Audio" +# diataxis: how-to +--- + +Mellea can send audio alongside your text prompt: pass it to any `instruct()` or `chat()` +call using the `audio` parameter. + +**Prerequisites:** an audio-capable model reachable through an OpenAI-compatible endpoint, +and an audio file. + +> **Backend note:** Only the OpenAI-compatible backends can send audio. `OllamaModelBackend`, +> `WatsonxAIBackend`, and `LocalHFBackend` raise a `ValueError` rather than silently dropping +> the clip — see [Backend support](#backend-support) below. + +--- + +## Basic usage + +`audio` takes a list of audio blocks. Build one with the constructor that matches your +source — `from_file` for a path on disk: + +```python +# Requires: mellea +# Returns: str +from mellea import start_session +from mellea.core import AudioBlock + +m = start_session("openai", model_id="gpt-audio-1.5") + +result = m.instruct( + "Transcribe the speech in this clip.", + audio=[AudioBlock.from_file("speech.wav")], +) +print(str(result)) +# Output will vary — LLM responses depend on model and temperature. +``` + +`audio` deliberately does **not** accept bare paths or URLs. Converting them is an +explicit step, so the type you pass says exactly what will be sent and any read or +download failure surfaces where you wrote it rather than mid-request. + +--- + +## Choosing a constructor + +| Source | Use | +| ------ | --- | +| A file on disk | `AudioBlock.from_file(path)` | +| Bytes in memory | `AudioBlock.from_bytes(data)` | +| A remote URL, fetched now | `AudioBlock.from_url(url)` | +| A remote URL, fetched at send time and cached | `AudioUrlBlock(url, format=...)` | +| Base64 you already have | `AudioBlock(value, format=...)` | + +`from_file`, `from_bytes`, and `from_url` all detect the format from the data's magic +bytes, so a mislabelled file is reported accurately — a WAV named `.mp3` yields +`format == "wav"`: + +```python +# Requires: mellea +# Returns: str +from mellea.core import AudioBlock + +clip = AudioBlock.from_file("speech.mp3") +print(clip.format) # "mp3" +``` + +```python +# Requires: mellea, requests +# Returns: AudioBlock +import requests +from mellea.core import AudioBlock + +wav = requests.get("https://cdn.openai.com/API/docs/audio/alloy.wav").content +clip = AudioBlock.from_bytes(wav) +``` + +Pass `format=` explicitly to skip detection when you already know it, or when the payload +is not one mellea recognises: + +```python +# Requires: mellea +# Returns: AudioBlock +from mellea.core import AudioBlock + +clip = AudioBlock.from_file("recording.opus", format="opus") +``` + +You can also construct a block from base64 directly. With a data URI the format is read from +the MIME type; with raw base64 you must supply `format`: + +```python +# Requires: mellea +# Returns: None +import base64 +from mellea.core import AudioBlock + +with open("speech.wav", "rb") as f: + b64 = base64.b64encode(f.read()).decode() + +from_data_uri = AudioBlock(f"data:audio/wav;base64,{b64}") # format inferred +from_raw = AudioBlock(b64, format="wav") # format required +``` + +### Remote audio + +No provider accepts audio by URL — OpenAI Chat Completions has no audio-by-URL content +part — so Mellea downloads the clip and inlines it for you. There are two ways, differing +only in *when* the fetch happens. + +`AudioBlock.from_url()` downloads immediately, so a bad URL fails at the call site: + +```python +# Requires: mellea +# Returns: AudioBlock +from mellea.core import AudioBlock + +clip = AudioBlock.from_url("https://example.com/speech.wav") +print(clip.format) # detected from the downloaded bytes +``` + +`AudioUrlBlock` defers the download to send time and memoizes it per URL, so a clip reused +across several turns is fetched once: + +```python +# Requires: mellea +# Returns: str +from mellea import start_session +from mellea.core import AudioUrlBlock + +m = start_session("openai", model_id="gpt-audio-1.5") + +clip = AudioUrlBlock("https://example.com/speech.wav", format="wav") +result = m.instruct("Transcribe this clip.", audio=[clip]) +print(str(result)) +# Output will vary — LLM responses depend on model and temperature. +``` + +Prefer `AudioUrlBlock` when the same URL is used repeatedly; prefer `from_url` when you +want the failure surfaced eagerly. Downloads are capped at 50 MB with a 30-second +timeout, and `AudioUrlBlock` requires an explicit `format` because nothing has been +fetched yet at construction time. + +--- + +## Supported formats + +OpenAI Chat Completions accepts only `wav` and `mp3` for audio input. Other +OpenAI-compatible servers may accept more, so mellea does not restrict the format it sends — +`flac` and `ogg` are detected and passed through, and an explicit `format=` is always +honoured. If a server rejects a format, that surfaces as a server-side error. + +**Mellea does not transcode audio.** Convert the file yourself (for example with `ffmpeg`), +or transcribe it to text and pass the transcript as part of your prompt. + +Format detection covers `wav`, `mp3`, `flac`, and `ogg`. When the bytes match none of these +and you did not pass `format=`, construction raises: + +```python +# Requires: mellea +# Returns: None +from mellea.core import AudioBlock + +try: + AudioBlock.from_file("notes.txt") +except ValueError as e: + print(e) + # Could not identify the audio format of 'notes.txt'. Pass format explicitly ... +``` + +--- + +## Multi-turn audio with ChatContext + +Audio passed to `instruct()` or `chat()` is stored in the +[`ChatContext`](../reference/glossary.md) turn history, so later calls in the same session +can refer back to the clip without passing it again: + +```python +# Requires: mellea +# Returns: None +from mellea import start_session +from mellea.core import AudioBlock +from mellea.stdlib.context import ChatContext + +m = start_session("openai", model_id="gpt-audio-1.5", ctx=ChatContext()) + +# First turn — attach the clip +r1 = m.instruct("Transcribe this clip.", audio=[AudioBlock.from_file("meeting.wav")]) +print(str(r1)) + +# Second turn — the clip is still in context +r2 = m.instruct("Summarise the main point in one sentence.") +print(str(r2)) +``` + +> **Cost warning:** the clip is re-sent on the wire on *every* subsequent turn, not just the +> first. Audio is far larger than text — a few minutes of WAV is megabytes of base64 and +> thousands of audio tokens — so a long conversation over one clip gets expensive quickly. +> For extended multi-turn work over the same audio, consider transcribing once and +> continuing over the transcript. + +--- + +## Backend support + +| Backend | Audio support | Notes | +| ------- | ------------- | ----- | +| `OpenAIBackend` | ✓ | Requires an audio-capable model | +| `LiteLLMBackend` | ✓ | Depends on the underlying provider and model | +| `OllamaModelBackend` | ✗ | Ollama's chat API has no audio input | +| `WatsonxAIBackend` | ✗ | The chat path carries no audio | +| `LocalHFBackend` | ✗ | Would require a processor-based audio model | + +The ✗ backends raise a `ValueError` when handed audio. This is deliberate: silently dropping +the clip would send a text-only prompt and produce a confident answer about audio the model +never received. + +> **Full example:** [`docs/examples/audio_text_models/audio_examples.py`](https://github.com/generative-computing/mellea/blob/main/docs/examples/audio_text_models/audio_examples.py) +> **Serving audio via `m serve`:** [`docs/examples/m_serve/multimodal-audio/`](https://github.com/generative-computing/mellea/tree/main/docs/examples/m_serve/multimodal-audio) + +--- + +**See also:** [Use Images and Vision Models](../how-to/use-images-and-vision.md) | +[Working with Data](../how-to/working-with-data.md) diff --git a/docs/sidebars.ts b/docs/sidebars.ts index fd02bfa1d..33460eed1 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -53,6 +53,7 @@ const sidebars: SidebarsConfig = { 'how-to/evaluate-with-llm-as-a-judge', 'how-to/configure-model-options', 'how-to/use-images-and-vision', + 'how-to/use-speech-and-audio', 'how-to/build-a-rag-pipeline', 'how-to/safety-guardrails', 'how-to/refactor-prompts-with-cli', diff --git a/mellea/backends/litellm.py b/mellea/backends/litellm.py index b2433bb38..186c0e19b 100644 --- a/mellea/backends/litellm.py +++ b/mellea/backends/litellm.py @@ -43,6 +43,7 @@ extract_model_tool_requests, get_current_event_loop, message_to_openai_message, + prefetch_audio_urls, send_to_queue, should_replay_reasoning, ) @@ -359,6 +360,7 @@ async def _generate_from_chat_context_standard( system_prompt = model_opts.get(ModelOption.SYSTEM_PROMPT, "") if system_prompt != "": conversation.append({"role": "system", "content": system_prompt}) + await prefetch_audio_urls(messages) replay_flags = should_replay_reasoning(messages, self._provider) conversation.extend( [ diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 657a0119f..e123b8b82 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -46,6 +46,7 @@ is_vllm_server_with_structured_output, message_to_openai_message, messages_to_docs, + prefetch_audio_urls, send_to_queue, should_replay_reasoning, ) @@ -1007,6 +1008,10 @@ async def _generate_from_chat_context_standard( conversation: list[dict] = [] + # Resolve any audio URLs off-thread so the sync serializer below hits the cache + # instead of blocking the event loop on a download. + await prefetch_audio_urls(messages) + system_prompt = model_opts.get(ModelOption.SYSTEM_PROMPT, "") if system_prompt != "": conversation.append({"role": "system", "content": system_prompt}) diff --git a/mellea/core/base.py b/mellea/core/base.py index f95464705..f042cabd0 100644 --- a/mellea/core/base.py +++ b/mellea/core/base.py @@ -331,6 +331,154 @@ def is_valid_base64_audio(s: str) -> bool: except (binascii.Error, ValueError): return False + @staticmethod + def detect_format(data: bytes) -> str | None: + """Identify an audio format from the magic bytes of raw (decoded) audio data. + + Detection is by content, not by file extension, so a mislabelled file is + reported accurately. Formats other than `wav` and `mp3` are still reported — + OpenAI-compatible endpoints accept only those two, but other backends may + accept more, matching the pass-through behaviour of the data-URI path. + + Args: + data (bytes): The raw, already-decoded audio bytes to inspect. + + Returns: + str | None: The detected format (e.g. `"wav"`), or `None` if the bytes + do not match a format this function recognises. + """ + # WAV / RIFF: form type must be WAVE, so a RIFF-wrapped AVI is not mistaken for audio. + if data[:4] == b"RIFF" and data[8:12] == b"WAVE": + return "wav" + if data[:4] == b"fLaC": + return "flac" + if data[:4] == b"OggS": + return "ogg" + # MP3: an ID3v2 tag, or a bare frame header whose 11-bit sync word is set. + if data[:3] == b"ID3": + return "mp3" + if len(data) >= 2 and data[0] == 0xFF and (data[1] & 0xE0) == 0xE0: + return "mp3" + return None + + @classmethod + def from_bytes( + cls, data: bytes, format: str | None = None, meta: dict[str, Any] | None = None + ) -> AudioBlock: + """Creates an `AudioBlock` from raw audio bytes, base64-encoding them. + + Args: + data (bytes): The raw audio bytes. + format (str | None): The audio format. When `None`, it is detected from + the data's magic bytes via `detect_format`. + meta (dict[str, Any] | None): Optional metadata to associate with the block. + + Returns: + AudioBlock: A new `AudioBlock` wrapping the base64-encoded audio. + + Raises: + ValueError: If `format` is `None` and the format cannot be detected from + the data. + """ + if format is None: + detected = cls.detect_format(data) + if detected is None: + raise ValueError( + "Could not detect the audio format from the data. " + "Pass format explicitly (e.g. format='wav') if you know it." + ) + format = detected + return cls(base64.b64encode(data).decode("utf-8"), format, meta) + + @classmethod + def from_file( + cls, + path: str | os.PathLike[str], + format: str | None = None, + meta: dict[str, Any] | None = None, + ) -> AudioBlock: + """Creates an `AudioBlock` by reading an audio file from disk. + + This saves callers from base64-encoding the file by hand. The format is + detected from the file's contents rather than its extension. + + Args: + path (str | os.PathLike[str]): Path to an audio file. + format (str | None): The audio format. When `None`, it is detected from + the file's magic bytes via `detect_format`. + meta (dict[str, Any] | None): Optional metadata to associate with the block. + + Returns: + AudioBlock: A new `AudioBlock` wrapping the base64-encoded file contents. + + Raises: + ValueError: If `path` cannot be opened (it is reported as a bad path rather + than surfacing a raw OS error, because `audio=` also accepts URLs and + base64 strings by mistake), or if `format` is `None` and the format + cannot be detected from the file's contents. + """ + try: + with open(path, "rb") as f: + data = f.read() + except OSError as e: + # `audio=` accepts bare strings, so a URL or a base64 payload lands here as + # FileNotFoundError / "File name too long". Say what was expected instead. + raise ValueError( + f"Could not read audio file {os.fspath(path)[:120]!r}: expected a path " + f"to an audio file on disk. ({type(e).__name__}: {e}) " + "To load remote audio, fetch the bytes and use AudioBlock.from_bytes(); " + "for base64 data use AudioBlock(value, format=...)." + ) from e + if format is None: + # Detect once here so the error can name the path, then hand the result to + # from_bytes rather than making it sniff the same bytes again. + format = cls.detect_format(data) + if format is None: + raise ValueError( + f"Could not identify the audio format of {os.fspath(path)!r}. " + "Pass format explicitly (e.g. format='wav') if you know it." + ) + return cls.from_bytes(data, format, meta) + + @classmethod + def from_url( + cls, url: str, format: str | None = None, meta: dict[str, Any] | None = None + ) -> AudioBlock: + """Creates an `AudioBlock` by downloading audio from a URL. + + Use this when you want the fetch to happen now and to fail here if it cannot. + Passing an `AudioUrlBlock` instead defers the same download to send time, where + it is memoized per URL — prefer that when the clip is reused across turns. + + Args: + url (str): An `http://` or `https://` URL pointing to an audio file. + format (str | None): The audio format. When `None`, it is detected from the + downloaded bytes' magic bytes via `detect_format`. + meta (dict[str, Any] | None): Optional metadata to associate with the block. + + Returns: + AudioBlock: A new `AudioBlock` wrapping the base64-encoded audio. + + Raises: + ValueError: If `url` is not an `http://`/`https://` URL, the download fails + or exceeds the size cap, or `format` is `None` and the format cannot be + detected from the downloaded bytes. + """ + if not url.startswith(("http://", "https://")): + raise ValueError( + f"AudioBlock.from_url requires an http:// or https:// URL; got: {url!r}. " + "For a local file use AudioBlock.from_file()." + ) + encoded = _cached_download_audio_as_base64(url) + if format is None: + format = cls.detect_format(base64.b64decode(encoded)) + if format is None: + raise ValueError( + f"Could not identify the audio format of the data at {url!r}. " + "Pass format explicitly (e.g. format='wav') if you know it." + ) + return cls(encoded, format, meta) + def __repr__(self) -> str: """Provides a python-parsable representation of the block (usually).""" return f"AudioBlock({self.value}, {self.format}, {self._meta.__repr__()})" @@ -339,6 +487,12 @@ def __repr__(self) -> str: class AudioUrlBlock(CBlock): """An `AudioUrlBlock` represents audio as a URL. + No provider accepts audio by URL — OpenAI Chat Completions has no audio-by-URL + content part — so backends resolve the URL to base64 on your behalf via + `resolve_base64`, mirroring how `ImageUrlBlock` is handled for backends that + require inline images. The download is memoized per URL, so re-sending the same + clip across conversation turns fetches it once. + Args: value (str): A URL string pointing to the audio. format (str): The audio format, such as `"wav"` or `"mp3"`. @@ -362,11 +516,126 @@ def __init__(self, value: str, format: str, meta: dict[str, Any] | None = None): super().__init__(value, meta) self.format = format + def resolve_base64(self) -> str: + """Return the audio as raw base64, downloading it once per URL. + + Providers require audio inline, so this is how an `AudioUrlBlock` reaches a + backend. The result is memoized in a process-wide, URL-keyed cache, so reusing + the URL across conversation turns does not re-fetch it. This call is blocking; + async callers should offload it with `asyncio.to_thread`. + + Returns: + str: The base64-encoded audio at the URL, with no data URI prefix. + + Raises: + ValueError: If the response exceeds the size cap, or the audio cannot be + downloaded. + """ + return _cached_download_audio_as_base64(str(self.value)) + def __repr__(self) -> str: """Provides a python-parsable representation of the block (usually).""" return f"AudioUrlBlock({self.value}, {self.format}, {self._meta.__repr__()})" +_AUDIO_DOWNLOAD_TIMEOUT_S: float = 30.0 +"""Socket timeout (seconds) applied to audio URL downloads. + +Longer than the image timeout: audio bodies are typically far larger. +""" + +_AUDIO_DOWNLOAD_MAX_BYTES: int = 50 * 1024 * 1024 +"""Maximum accepted size (bytes) of a downloaded audio body.""" + +_AUDIO_CACHE_MAX_ENTRIES: int = 32 +"""Maximum number of URL -> base64 entries retained by the audio download cache. + +Smaller than the image cache because audio payloads are much larger per entry. +""" + +_audio_base64_cache: OrderedDict[str, str] = OrderedDict() +"""Process-wide LRU cache mapping audio URLs to their base64-encoded payload.""" + +_audio_base64_cache_lock = threading.Lock() + + +def _cached_download_audio_as_base64(url: str) -> str: + """Download audio as base64, memoizing the result per URL. + + Mirrors `_cached_download_image_as_base64`: the download runs outside the lock so + concurrent fetches of distinct URLs proceed in parallel, and only the small cache + read/write is serialized. + + Args: + url: An `http://` or `https://` URL pointing to an audio file. + + Returns: + str: The base64-encoded audio at the URL. + + Raises: + ValueError: If the response exceeds the size cap or the audio cannot be + downloaded. + """ + with _audio_base64_cache_lock: + cached = _audio_base64_cache.get(url) + if cached is not None: + _audio_base64_cache.move_to_end(url) # mark as most-recently used + return cached + + encoded = _download_audio_as_base64(url) + + with _audio_base64_cache_lock: + _audio_base64_cache[url] = encoded + _audio_base64_cache.move_to_end(url) + while len(_audio_base64_cache) > _AUDIO_CACHE_MAX_ENTRIES: + _audio_base64_cache.popitem(last=False) # evict least-recently used + return encoded + + +def _download_audio_as_base64(url: str) -> str: + """Download audio from a URL and return it base64-encoded. + + Unlike the image equivalent there is no decode/re-encode step: audio bytes are + passed through as-is, so the caller's declared format stays authoritative and no + transcoding is implied. + + A timeout bounds slow responses and the body is streamed with a size cap to guard + against memory exhaustion. This function is blocking; async callers should offload + it with `asyncio.to_thread`. + + Args: + url: An `http://` or `https://` URL pointing to an audio file. + + Returns: + str: The base64-encoded audio bytes. + + Raises: + ValueError: If the response exceeds the size cap or the audio cannot be + downloaded. + """ + try: + with requests.get( # scheme validated by caller + url, timeout=_AUDIO_DOWNLOAD_TIMEOUT_S, stream=True + ) as response: + response.raise_for_status() + declared = response.headers.get("Content-Length") + if declared is not None and int(declared) > _AUDIO_DOWNLOAD_MAX_BYTES: + raise ValueError( + f"Audio at {url!r} exceeds the {_AUDIO_DOWNLOAD_MAX_BYTES}-byte limit" + ) + # Stream so an undeclared/lying Content-Length can't exhaust memory. + raw = response.raw.read(_AUDIO_DOWNLOAD_MAX_BYTES + 1, decode_content=True) + if len(raw) > _AUDIO_DOWNLOAD_MAX_BYTES: + raise ValueError( + f"Audio at {url!r} exceeds the {_AUDIO_DOWNLOAD_MAX_BYTES}-byte limit" + ) + if not raw: + raise ValueError(f"Audio at {url!r} was empty") + except (requests.RequestException, OSError, ValueError) as e: + raise ValueError(f"Failed to download audio from URL {url!r}: {e}") from e + return base64.b64encode(raw).decode("utf-8") + + _IMAGE_DOWNLOAD_TIMEOUT_S: float = 10.0 """Socket timeout (seconds) applied to image URL downloads.""" @@ -2087,57 +2356,116 @@ def blockify(s: str | Span) -> Span: raise Exception("Type Error") +def _attachments_from_representation( + c: Component, field: str, expected: tuple[type, ...] +) -> None | list: + """Read an attachment list off a component's `TemplateRepresentation`. + + A component can carry attachments two ways: as an attribute (`Message.audio`, + `Instruction.images`) or purely by declaring them on the `TemplateRepresentation` + its `format_for_llm` returns. Attribute-less components are invisible to the + attribute check, so backends that reject unsupported modalities would drop their + attachments silently — the failure this fallback exists to prevent. + + Only called when the attribute is absent, so components that expose one (every + built-in carrier does) never pay for the extra `format_for_llm` call. + + Args: + c: The `Component` whose representation is inspected. + field: The `TemplateRepresentation` field to read — `"images"` or `"audio"`. + expected: The block types every element must be an instance of. Validated here + so the declared return type is not a lie the backend discovers later. + + Returns: + The non-empty attachment list declared on the representation, or `None` when + the component declares none, does not return a `TemplateRepresentation`, or + raises while building one. + + Raises: + AssertionError: If the declared attachments are not a list, or any element is + not an instance of one of `expected`. + """ + try: + tr = c.format_for_llm() + except Exception as e: + # A guard must not become a new source of failure; treat an unrenderable + # component as carrying nothing and let the real render surface the error. + # Logged so a genuinely broken format_for_llm is not silently invisible. + logging.getLogger(__name__).debug( + f"could not read {field} from {type(c).__name__}.format_for_llm(); " + f"treating it as carrying no {field}: {e!r}" + ) + return None + if not isinstance(tr, TemplateRepresentation): + return None + attachments = getattr(tr, field, None) + if not attachments: + return None + assert isinstance(attachments, list), ( + f"TemplateRepresentation.{field} must be a list." + ) + assert all(isinstance(a, expected) for a in attachments), ( + f"all elements of TemplateRepresentation.{field} must be one of: " + f"{', '.join(t.__name__ for t in expected)}." + ) + return list(attachments) + + def get_images_from_component(c: Component) -> None | list[ImageBlock | ImageUrlBlock]: """Return the images attached to a `Component`, or `None` if absent or empty. + Checks the component's `images` attribute first. When it has none, falls back to + the `images` declared on the `TemplateRepresentation` returned by + `format_for_llm`, so components that only declare attachments there are still + visible to backend capability checks. + Args: c: The `Component` whose `images` attribute is inspected. Returns: A non-empty list of `ImageBlock` or `ImageUrlBlock` objects if the - component has an `images` attribute with at least one element; - `None` otherwise. + component has an `images` attribute with at least one element, or declares + images on its `TemplateRepresentation`; `None` otherwise. """ - if hasattr(c, "images"): - imgs = c.images # type: ignore - if imgs is not None: - assert isinstance(imgs, list), "images field must be a list." - assert all(isinstance(im, (ImageBlock, ImageUrlBlock)) for im in imgs), ( - "all elements of images list must be ImageBlock or ImageUrlBlock." - ) - if len(imgs) == 0: - return None - else: - return imgs - else: - return None - else: + if not hasattr(c, "images"): + return _attachments_from_representation( + c, "images", (ImageBlock, ImageUrlBlock) + ) + + imgs = c.images # type: ignore + if imgs is None: return None + assert isinstance(imgs, list), "images field must be a list." + assert all(isinstance(im, (ImageBlock, ImageUrlBlock)) for im in imgs), ( + "all elements of images list must be ImageBlock or ImageUrlBlock." + ) + return imgs if len(imgs) > 0 else None def get_audio_from_component(c: Component) -> None | list[AudioBlock | AudioUrlBlock]: """Return the audio attached to a `Component`, or `None` if absent or empty. + Checks the component's `audio` attribute first. When it has none, falls back to + the `audio` declared on the `TemplateRepresentation` returned by `format_for_llm`, + so components that only declare attachments there are still visible to backend + capability checks. + Args: c: The `Component` whose `audio` attribute is inspected. Returns: A non-empty list of `AudioBlock` or `AudioUrlBlock` objects if the - component has an `audio` attribute with at least one element; - `None` otherwise. + component has an `audio` attribute with at least one element, or declares + audio on its `TemplateRepresentation`; `None` otherwise. """ - if hasattr(c, "audio"): - audio = c.audio # type: ignore - if audio is not None: - assert isinstance(audio, list), "audio field must be a list." - assert all(isinstance(a, (AudioBlock, AudioUrlBlock)) for a in audio), ( - "all elements of audio list must be AudioBlock or AudioUrlBlock." - ) - if len(audio) == 0: - return None - else: - return audio - else: - return None - else: + if not hasattr(c, "audio"): + return _attachments_from_representation(c, "audio", (AudioBlock, AudioUrlBlock)) + + audio = c.audio # type: ignore + if audio is None: return None + assert isinstance(audio, list), "audio field must be a list." + assert all(isinstance(a, (AudioBlock, AudioUrlBlock)) for a in audio), ( + "all elements of audio list must be AudioBlock or AudioUrlBlock." + ) + return audio if len(audio) > 0 else None diff --git a/mellea/helpers/__init__.py b/mellea/helpers/__init__.py index 8dfd7c0ae..714c0e4cc 100644 --- a/mellea/helpers/__init__.py +++ b/mellea/helpers/__init__.py @@ -27,6 +27,7 @@ merge_provider_fields, message_to_openai_message, messages_to_docs, + prefetch_audio_urls, should_replay_reasoning, ) from .server_type import ( @@ -49,6 +50,7 @@ "merge_provider_fields", "message_to_openai_message", "messages_to_docs", + "prefetch_audio_urls", "send_to_queue", "should_replay_reasoning", "wait_for_all_mots", diff --git a/mellea/helpers/openai_compatible_helpers.py b/mellea/helpers/openai_compatible_helpers.py index c0dd7f5eb..394b921a7 100644 --- a/mellea/helpers/openai_compatible_helpers.py +++ b/mellea/helpers/openai_compatible_helpers.py @@ -5,6 +5,7 @@ from __future__ import annotations +import asyncio import copy import json import uuid @@ -354,6 +355,34 @@ def should_replay_reasoning( return flags +async def prefetch_audio_urls(messages: list[Message]) -> None: + """Warm the audio download cache for any `AudioUrlBlock` in `messages`. + + No provider accepts audio by URL, so `message_to_openai_message` resolves such + blocks to inline base64. That resolution is blocking, and the serializer is sync; + awaiting this first moves the fetch onto a worker thread so the event loop is not + stalled, leaving the serializer's call a cache hit. Mirrors how the Ollama backend + offloads `ImageUrlBlock` downloads with `asyncio.to_thread`. + + Safe to call when there is nothing to fetch, and safe to skip — the serializer still + produces correct output either way, just with a blocking download. + + Args: + messages: The messages about to be serialised. + + Raises: + ValueError: If a URL cannot be downloaded or exceeds the size cap. + """ + pending = [ + a for m in messages for a in (m.audio or []) if isinstance(a, AudioUrlBlock) + ] + if not pending: + return + await asyncio.gather( + *(asyncio.to_thread(a.resolve_base64) for a in pending), return_exceptions=False + ) + + def message_to_openai_message( msg: Message, formatter: Formatter | None = None, @@ -435,12 +464,19 @@ def message_to_openai_message( } ) elif isinstance(audio, AudioUrlBlock): - # OpenAI Chat Completions does not support audio by URL; - # AudioUrlBlock cannot be serialised to this schema. - raise ValueError( - f"AudioUrlBlock cannot be serialised to the OpenAI Chat Completions " - f"audio schema (URL: {audio.value!r}). " - "Fetch the audio and use AudioBlock with base64 data instead." + # OpenAI Chat Completions has no audio-by-URL content part, so + # resolve it to inline base64 the way Ollama does for images. + # Normally a cache hit: `prefetch_audio_urls` warms it off-thread + # before serialisation. Falls back to a blocking fetch if some + # caller path did not prefetch. + parts.append( + { + "type": "input_audio", + "input_audio": { + "data": audio.resolve_base64(), + "format": audio.format, + }, + } ) result: dict[str, Any] = {"role": msg.role, "content": parts} diff --git a/test/backends/test_audio_openai.py b/test/backends/test_audio_openai.py index 5c21131f8..564d4e86d 100644 --- a/test/backends/test_audio_openai.py +++ b/test/backends/test_audio_openai.py @@ -55,7 +55,7 @@ from mellea import MelleaSession, start_session from mellea.backends import ModelOption -from mellea.core import AudioBlock, ModelOutputThunk +from mellea.core import AudioBlock, AudioUrlBlock, ModelOutputThunk from mellea.stdlib.components import Message @@ -85,14 +85,60 @@ class AudioContent(BaseModel): @pytest.fixture(scope="module") -def sample_audio_wav() -> str: - """Download the Gemma sample speech WAV ('Roses are red, violets are blue.').""" +def sample_audio_bytes() -> bytes: + """Download the raw Gemma sample speech WAV ('Roses are red, violets are blue.').""" response = requests.get(_AUDIO_URL, timeout=30) response.raise_for_status() - encoded = base64.b64encode(response.content).decode("utf-8") + return response.content + + +@pytest.fixture(scope="module") +def sample_audio_wav(sample_audio_bytes: bytes) -> str: + """The Gemma sample speech WAV as a base64 data URI.""" + encoded = base64.b64encode(sample_audio_bytes).decode("utf-8") return f"data:audio/wav;base64,{encoded}" +@pytest.fixture(scope="module") +def sample_audio_path(tmp_path_factory, sample_audio_bytes: bytes): + """The Gemma sample speech WAV written to a real file on disk. + + Deliberately given a `.bin` extension so tests that load it exercise + content-based format detection rather than trusting the extension. + """ + path = tmp_path_factory.mktemp("audio") / "roses-are.bin" + path.write_bytes(sample_audio_bytes) + return path + + +def _assert_recognised_roses_and_violets(result) -> None: + """Assert the model heard 'Roses are red, violets are blue.' in the clip. + + The prompt names neither a colour nor a flower, so these values can only come + from the audio actually reaching the model and being understood. + + Args: + result: The `ModelOutputThunk` whose value is `AudioContent` JSON. + """ + assert isinstance(result, ModelOutputThunk) + assert result.value is not None + parsed = AudioContent.model_validate_json(result.value) + colors = [c.lower() for c in parsed.colors] + flowers = [f.lower() for f in parsed.flowers] + assert any("red" in c or c == "red" for c in colors), ( + f"Expected 'red' in colors, got: {parsed.colors!r}" + ) + assert any("blue" in c or c == "blue" for c in colors), ( + f"Expected 'blue' in colors, got: {parsed.colors!r}" + ) + assert any("rose" in f for f in flowers), ( + f"Expected 'rose(s)' in flowers, got: {parsed.flowers!r}" + ) + assert any("violet" in f for f in flowers), ( + f"Expected 'violet(s)' in flowers, got: {parsed.flowers!r}" + ) + + def test_audio_block_construction(sample_audio_wav: str): """Test that AudioBlock can be constructed from base64 WAV data.""" audio_block = AudioBlock(sample_audio_wav) @@ -250,5 +296,86 @@ def test_session_chat_with_audio(sample_audio_wav: str): assert len(turn.model_input.audio) > 0 +# --- File-loading paths, verified end to end --- +# +# The tests above build an AudioBlock from a base64 data URI. These cover the +# from_file / bare-path entry points instead, so a regression in magic-byte +# detection or in the path coercion is caught by a test that only passes when the +# model actually understood the clip -- not merely when the payload was well-formed. + + +@pytest.mark.qualitative +def test_audio_from_file_reaches_model(sample_audio_path): + """A clip loaded with AudioBlock.from_file() is understood by the model. + + Exercises detect_format -> from_file -> input_audio -> comprehension. The file has + a `.bin` extension, so passing also proves the format was detected from content. + """ + audio_block = AudioBlock.from_file(sample_audio_path) + assert audio_block.format == "wav", ( + "format must be detected from content, not the .bin extension" + ) + + with _make_session() as session: + result = session.instruct( + "Listen to the audio and return the colors and flowers mentioned.", + audio=[audio_block], + strategy=None, + format=AudioContent, + ) + _assert_recognised_roses_and_violets(result) + + +@pytest.mark.qualitative +def test_audio_url_block_reaches_model(): + """An AudioUrlBlock is downloaded at send time and understood by the model. + + Unlike the mocked unit tests, this performs the real fetch, so it also proves the + download-and-inline path works end to end against a live provider. + """ + clip = AudioUrlBlock(_AUDIO_URL, format="wav") + + with _make_session() as session: + result = session.instruct( + "Listen to the audio and return the colors and flowers mentioned.", + audio=[clip], + strategy=None, + format=AudioContent, + ) + _assert_recognised_roses_and_violets(result) + + +@pytest.mark.qualitative +def test_audio_from_url_reaches_model(): + """A clip built with AudioBlock.from_url() is understood by the model.""" + audio_block = AudioBlock.from_url(_AUDIO_URL) + assert audio_block.format == "wav", "format must be detected from downloaded bytes" + + with _make_session() as session: + result = session.instruct( + "Listen to the audio and return the colors and flowers mentioned.", + audio=[audio_block], + strategy=None, + format=AudioContent, + ) + _assert_recognised_roses_and_violets(result) + + +@pytest.mark.qualitative +def test_audio_from_bytes_reaches_model(sample_audio_bytes: bytes): + """A clip built with AudioBlock.from_bytes() is understood by the model.""" + audio_block = AudioBlock.from_bytes(sample_audio_bytes) + assert audio_block.format == "wav" + + with _make_session() as session: + result = session.instruct( + "Listen to the audio and return the colors and flowers mentioned.", + audio=[audio_block], + strategy=None, + format=AudioContent, + ) + _assert_recognised_roses_and_violets(result) + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/backends/test_audio_openai_unit.py b/test/backends/test_audio_openai_unit.py index 6e32d93e1..2fb9e90f9 100644 --- a/test/backends/test_audio_openai_unit.py +++ b/test/backends/test_audio_openai_unit.py @@ -125,9 +125,26 @@ def test_audio_block_in_instruct_payload_shape( ) -def test_audio_url_block_rejected_by_openai(mocked_openai_session: tuple): - """AudioUrlBlock raises ValueError before the request is sent.""" - session, _ = mocked_openai_session +def test_audio_url_block_downloaded_and_inlined_by_openai(mocked_openai_session: tuple): + """AudioUrlBlock is fetched and sent as an inline input_audio part. + + OpenAI Chat Completions has no audio-by-URL content part, so the URL is resolved + rather than rejected — mirroring how Ollama inlines `ImageUrlBlock`. The download is + patched out, so this asserts the wiring rather than the network. + """ + session, mock_client = mocked_openai_session url_block = AudioUrlBlock("https://example.com/audio.wav", format="wav") - with pytest.raises(ValueError, match="AudioUrlBlock"): + + with patch.object(AudioUrlBlock, "resolve_base64", return_value=_B64_WAV): session.instruct("Transcribe this.", audio=[url_block], strategy=None) + + messages = mock_client.chat.completions.create.await_args.kwargs["messages"] + audio_parts = [ + part + for m in messages + if isinstance(m.get("content"), list) + for part in m["content"] + if part.get("type") == "input_audio" + ] + assert len(audio_parts) == 1 + assert audio_parts[0]["input_audio"] == {"data": _B64_WAV, "format": "wav"} diff --git a/test/backends/test_huggingface_unit.py b/test/backends/test_huggingface_unit.py index 68343fe5b..f8ce004c2 100644 --- a/test/backends/test_huggingface_unit.py +++ b/test/backends/test_huggingface_unit.py @@ -1573,6 +1573,41 @@ async def test_multimodal_blocks_in_raw_action_raises_error(images, audio): await backend._generate_from_raw([action], ctx, model_options={}) +@pytest.mark.asyncio +async def test_multimodal_declared_only_on_representation_raises_error(): + """Audio declared solely on a TemplateRepresentation must still be rejected. + + A custom component may carry attachments without exposing an `audio`/`images` + attribute. Such components are invisible to the attribute check, so without the + representation fallback in `get_audio_from_component` the clip would be dropped + silently and the model would answer about audio it never received. + """ + from mellea.core import AudioBlock, Component, TemplateRepresentation + + class _AudioOnRepresentation(Component): + def parts(self): + return [] + + def format_for_llm(self) -> TemplateRepresentation: + return TemplateRepresentation( + obj=self, + args={"t": "rate this"}, + template="{{ t }}", + audio=[AudioBlock(_B64_WAV, format="wav")], + ) + + def _parse(self, computed): + return str(computed.value) + + backend = _make_backend() + ctx = ChatContext().add(Message("user", "Hello")) + + with pytest.raises(ValueError, match="LocalHFBackend does not support audio"): + await backend._generate_from_context_standard( + _AudioOnRepresentation(), ctx, model_options={} + ) + + @pytest.mark.parametrize("images,audio", _MULTIMODAL_CASES) @pytest.mark.asyncio async def test_multimodal_blocks_in_raw_ctx_not_checked(images, audio): diff --git a/test/core/test_base.py b/test/core/test_base.py index 6038e714c..127aeae95 100644 --- a/test/core/test_base.py +++ b/test/core/test_base.py @@ -4,7 +4,9 @@ import base64 import copy import io +import wave from typing import Any +from unittest.mock import patch import pytest from PIL import Image as PILImage @@ -20,8 +22,10 @@ ModelOutputThunk, ModelToolCall, RawProviderResponse, + TemplateRepresentation, blockify, get_audio_from_component, + get_images_from_component, make_image_block, ) from mellea.core.backend import generate_walk @@ -254,6 +258,194 @@ def test_audio_block_missing_format_raises(): AudioBlock(raw_b64) +# --- AudioBlock.detect_format / from_bytes / from_file --- + + +def _make_wav_bytes(seconds: float = 0.05, rate: int = 8000) -> bytes: + """Return a minimal mono 16-bit PCM WAV via the stdlib `wave` module.""" + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(rate) + w.writeframes(b"\x00\x10" * int(rate * seconds)) + return buf.getvalue() + + +@pytest.mark.parametrize( + "data,expected", + [ + (b"ID3\x04\x00\x00\x00", "mp3"), # ID3v2 tag + (b"\xff\xfb\x90\x00", "mp3"), # bare frame sync + (b"fLaC\x00\x00\x00\x00", "flac"), + (b"OggS\x00\x02\x00\x00", "ogg"), + ], +) +def test_audio_detect_format_by_magic_bytes(data: bytes, expected: str): + assert AudioBlock.detect_format(data) == expected + + +def test_audio_detect_format_wav(): + assert AudioBlock.detect_format(_make_wav_bytes()) == "wav" + + +def test_audio_detect_format_riff_without_wave_is_not_audio(): + """A RIFF container that is not a WAVE form (e.g. AVI) must not pass as wav.""" + assert AudioBlock.detect_format(b"RIFF\x00\x00\x00\x00AVI LIST") is None + + +@pytest.mark.parametrize("data", [b"", b"nope", b"\x01\x02\x03\x04"]) +def test_audio_detect_format_unrecognised(data: bytes): + assert AudioBlock.detect_format(data) is None + + +def test_audio_from_bytes_detects_format(): + wav = _make_wav_bytes() + block = AudioBlock.from_bytes(wav) + assert block.format == "wav" + assert base64.b64decode(str(block.value)) == wav + + +def test_audio_from_bytes_explicit_format_overrides_detection(): + """An explicit format is trusted, so unrecognised payloads remain usable.""" + block = AudioBlock.from_bytes(b"\x01\x02\x03\x04", "wav") + assert block.format == "wav" + + +def test_audio_from_bytes_undetectable_raises(): + with pytest.raises(ValueError, match="Could not detect the audio format"): + AudioBlock.from_bytes(b"\x01\x02\x03\x04") + + +def test_audio_from_file(tmp_path): + wav = _make_wav_bytes() + path = tmp_path / "clip.wav" + path.write_bytes(wav) + block = AudioBlock.from_file(path) + assert block.format == "wav" + assert base64.b64decode(str(block.value)) == wav + + +def test_audio_from_file_accepts_str_path(tmp_path): + path = tmp_path / "clip.wav" + path.write_bytes(_make_wav_bytes()) + assert AudioBlock.from_file(str(path)).format == "wav" + + +def test_audio_from_file_detects_by_content_not_extension(tmp_path): + """A WAV named .mp3 is reported as wav — extensions are not trusted.""" + path = tmp_path / "mislabelled.mp3" + path.write_bytes(_make_wav_bytes()) + assert AudioBlock.from_file(path).format == "wav" + + +def test_audio_from_file_unrecognised_format_raises(tmp_path): + path = tmp_path / "clip.bin" + path.write_bytes(b"\x01\x02\x03\x04") + with pytest.raises(ValueError, match="Could not identify the audio format"): + AudioBlock.from_file(path) + + +def test_audio_from_file_explicit_format_skips_detection(tmp_path): + path = tmp_path / "clip.bin" + path.write_bytes(b"\x01\x02\x03\x04") + assert AudioBlock.from_file(path, "wav").format == "wav" + + +def test_audio_from_file_missing_raises(tmp_path): + with pytest.raises(ValueError, match="expected a path to an audio file"): + AudioBlock.from_file(tmp_path / "nope.wav") + + +@pytest.mark.parametrize( + "not_a_path", + [ + "https://example.com/clip.wav", # a URL — plausible mistake + base64.b64encode(b"\x00" * 400).decode(), # base64 payload — "name too long" + ], +) +def test_audio_from_file_non_path_string_gives_actionable_error(not_a_path: str): + """`audio=` accepts bare strings, so URLs and base64 land here; say what was wanted. + + Without this the caller sees a raw FileNotFoundError or + `OSError: File name too long`, neither of which names the real mistake. + """ + with pytest.raises(ValueError, match="expected a path to an audio file") as exc: + AudioBlock.from_file(not_a_path) + assert "from_bytes" in str(exc.value) + + +# --- AudioBlock.from_url / AudioUrlBlock.resolve_base64 --- +# +# No provider accepts audio by URL, so a URL must be resolved to inline base64 — +# eagerly via from_url, or lazily at send time via AudioUrlBlock.resolve_base64. +# The network is patched out throughout; these assert wiring, not connectivity. + + +def test_audio_from_url_downloads_and_detects_format(): + wav = _make_wav_bytes() + encoded = base64.b64encode(wav).decode() + with patch( + "mellea.core.base._cached_download_audio_as_base64", return_value=encoded + ) as mock_dl: + block = AudioBlock.from_url("https://example.com/clip.wav") + mock_dl.assert_called_once_with("https://example.com/clip.wav") + assert block.format == "wav" + assert block.value == encoded + + +def test_audio_from_url_explicit_format_skips_detection(): + encoded = base64.b64encode(b"\x01\x02\x03\x04").decode() + with patch( + "mellea.core.base._cached_download_audio_as_base64", return_value=encoded + ): + assert AudioBlock.from_url("https://example.com/x.bin", "wav").format == "wav" + + +def test_audio_from_url_undetectable_format_raises(): + encoded = base64.b64encode(b"\x01\x02\x03\x04").decode() + with patch( + "mellea.core.base._cached_download_audio_as_base64", return_value=encoded + ): + with pytest.raises(ValueError, match="Could not identify the audio format"): + AudioBlock.from_url("https://example.com/x.bin") + + +@pytest.mark.parametrize("bad", ["clip.wav", "ftp://example.com/a.wav", "/tmp/a.wav"]) +def test_audio_from_url_rejects_non_http(bad: str): + """A local path here is a plausible mistake; point at from_file rather than fetching.""" + with pytest.raises(ValueError, match="requires an http:// or https:// URL") as exc: + AudioBlock.from_url(bad) + assert "from_file" in str(exc.value) + + +def test_audio_url_block_resolve_base64_delegates_to_cache(): + encoded = base64.b64encode(_make_wav_bytes()).decode() + block = AudioUrlBlock("https://example.com/clip.wav", format="wav") + with patch( + "mellea.core.base._cached_download_audio_as_base64", return_value=encoded + ) as mock_dl: + assert block.resolve_base64() == encoded + mock_dl.assert_called_once_with("https://example.com/clip.wav") + + +def test_audio_download_cache_fetches_once_per_url(): + """Reusing a URL across turns must not re-fetch it.""" + from mellea.core.base import _audio_base64_cache, _cached_download_audio_as_base64 + + url = "https://example.com/cached-once.wav" + _audio_base64_cache.pop(url, None) + encoded = base64.b64encode(_make_wav_bytes()).decode() + with patch( + "mellea.core.base._download_audio_as_base64", return_value=encoded + ) as mock_dl: + first = _cached_download_audio_as_base64(url) + second = _cached_download_audio_as_base64(url) + assert first == second == encoded + mock_dl.assert_called_once() + _audio_base64_cache.pop(url, None) + + # --- AudioUrlBlock --- @@ -337,6 +529,113 @@ def _parse(self, computed: ModelOutputThunk) -> str: assert get_audio_from_component(_ComponentWithoutAudio()) is None +# --- TemplateRepresentation fallback for attribute-less components --- +# +# A component may declare attachments only on its TemplateRepresentation. Without a +# fallback such components are invisible to backend capability checks, so backends +# that cannot carry the modality (e.g. LocalHFBackend) would drop them silently. + + +class _ComponentDeclaringOnRepresentation(Component[str]): + """Declares attachments only via format_for_llm — deliberately has no attribute.""" + + def __init__(self, *, images=None, audio=None): + self._declared_images = images + self._declared_audio = audio + self.format_for_llm_calls = 0 + + def parts(self): + return [] + + def format_for_llm(self) -> TemplateRepresentation: + self.format_for_llm_calls += 1 + return TemplateRepresentation( + obj=self, args={}, images=self._declared_images, audio=self._declared_audio + ) + + def _parse(self, computed: ModelOutputThunk) -> str: + return "" + + +def test_get_audio_falls_back_to_template_representation(): + audio = [AudioBlock(base64.b64encode(b"audio").decode(), format="wav")] + component = _ComponentDeclaringOnRepresentation(audio=audio) + assert get_audio_from_component(component) == audio + + +def test_get_images_falls_back_to_template_representation(): + images = [ImageUrlBlock("https://example.com/cat.png")] + component = _ComponentDeclaringOnRepresentation(images=images) + assert get_images_from_component(component) == images + + +def test_representation_fallback_returns_none_when_nothing_declared(): + component = _ComponentDeclaringOnRepresentation() + assert get_audio_from_component(component) is None + assert get_images_from_component(component) is None + + +def test_representation_fallback_validates_element_types(): + """The fallback must not return non-blocks under a block-typed signature. + + Without validation the declared return type is a lie the backend discovers later. + """ + bad_audio = _ComponentDeclaringOnRepresentation(audio=["not-a-block"]) + with pytest.raises(AssertionError, match=r"TemplateRepresentation\.audio"): + get_audio_from_component(bad_audio) + + bad_images = _ComponentDeclaringOnRepresentation(images=[object()]) + with pytest.raises(AssertionError, match=r"TemplateRepresentation\.images"): + get_images_from_component(bad_images) + + +def test_representation_fallback_not_consulted_when_attribute_present(): + """Components exposing the attribute must not pay for an extra format_for_llm call.""" + + class _WithAttributeAndRepresentation(_ComponentDeclaringOnRepresentation): + audio = None # attribute present, so the fallback must be skipped + + component = _WithAttributeAndRepresentation( + audio=[AudioBlock(base64.b64encode(b"audio").decode(), format="wav")] + ) + assert get_audio_from_component(component) is None + assert component.format_for_llm_calls == 0 + + +def test_representation_fallback_swallows_format_for_llm_errors(): + """A capability guard must not become a new source of failure.""" + + class _Exploding(Component[str]): + def parts(self): + return [] + + def format_for_llm(self) -> TemplateRepresentation: + raise RuntimeError("cannot render") + + def _parse(self, computed: ModelOutputThunk) -> str: + return "" + + assert get_audio_from_component(_Exploding()) is None + assert get_images_from_component(_Exploding()) is None + + +def test_representation_fallback_ignores_string_representation(): + """format_for_llm may return a plain string; that carries no attachments.""" + + class _StringRepr(Component[str]): + def parts(self): + return [] + + def format_for_llm(self) -> str: + return "just text" + + def _parse(self, computed: ModelOutputThunk) -> str: + return "" + + assert get_audio_from_component(_StringRepr()) is None + assert get_images_from_component(_StringRepr()) is None + + # --- make_image_block factory --- diff --git a/test/helpers/test_openai_compatible_helpers.py b/test/helpers/test_openai_compatible_helpers.py index 1a1be5a77..9261d62e0 100644 --- a/test/helpers/test_openai_compatible_helpers.py +++ b/test/helpers/test_openai_compatible_helpers.py @@ -7,6 +7,7 @@ import json from datetime import datetime from decimal import Decimal +from unittest.mock import patch import pytest @@ -25,6 +26,7 @@ extract_model_tool_requests, message_to_openai_message, messages_to_docs, + prefetch_audio_urls, ) from mellea.stdlib.components import Document, Message, ToolMessage @@ -533,12 +535,52 @@ def test_audio_only_no_images(self): assert result["content"][0]["type"] == "text" assert result["content"][1]["type"] == "input_audio" - def test_audio_url_block_raises(self): - """AudioUrlBlock cannot be serialised to the OpenAI schema and must raise.""" + def test_audio_url_block_resolved_to_inline_base64(self): + """AudioUrlBlock is downloaded and inlined, mirroring ImageUrlBlock for Ollama. + + OpenAI Chat Completions has no audio-by-URL content part, so the URL is resolved + rather than rejected. The download is patched out: this asserts the wiring, not + the network. + """ audio_url = AudioUrlBlock("https://example.com/audio.wav", format="wav") msg = Message(role="user", content="listen", audio=[audio_url]) - with pytest.raises(ValueError, match="AudioUrlBlock"): - message_to_openai_message(msg) + + with patch.object( + AudioUrlBlock, "resolve_base64", return_value=_B64_WAV + ) as mock_resolve: + result = message_to_openai_message(msg) + + mock_resolve.assert_called_once() + part = result["content"][1] + assert part["type"] == "input_audio" + assert part["input_audio"] == {"data": _B64_WAV, "format": "wav"} + + async def test_prefetch_audio_urls_warms_cache_off_thread(self): + """prefetch_audio_urls resolves every URL block so the sync serializer is a hit.""" + blocks = [ + AudioUrlBlock("https://example.com/a.wav", format="wav"), + AudioUrlBlock("https://example.com/b.mp3", format="mp3"), + ] + msgs = [Message(role="user", content="x", audio=blocks)] + + with patch.object( + AudioUrlBlock, "resolve_base64", return_value=_B64_WAV + ) as mock_resolve: + await prefetch_audio_urls(msgs) + + assert mock_resolve.call_count == 2 + + async def test_prefetch_audio_urls_noop_without_url_blocks(self): + """Nothing to fetch must not touch resolve_base64 at all.""" + msgs = [ + Message( + role="user", content="x", audio=[AudioBlock(_B64_WAV, format="wav")] + ), + Message(role="user", content="y"), + ] + with patch.object(AudioUrlBlock, "resolve_base64") as mock_resolve: + await prefetch_audio_urls(msgs) + mock_resolve.assert_not_called() def test_empty_audio_list(self): """Empty audio list triggers multimodal content format (unlike None)."""