diff --git a/mellea/backends/adapters/__init__.py b/mellea/backends/adapters/__init__.py index 3055e77e2..932f3f55b 100644 --- a/mellea/backends/adapters/__init__.py +++ b/mellea/backends/adapters/__init__.py @@ -25,6 +25,7 @@ ) from .capabilities import KNOWN_CAPABILITIES from .catalog import validate_revision +from .io_contracts import get_io_contract __all__ = [ "KNOWN_CAPABILITIES", @@ -44,5 +45,6 @@ "WeightsBinding", "fetch_intrinsic_metadata", "get_adapter_for_intrinsic", + "get_io_contract", "validate_revision", ] diff --git a/mellea/backends/adapters/_core.py b/mellea/backends/adapters/_core.py index 5351d12f6..0552f5bfd 100644 --- a/mellea/backends/adapters/_core.py +++ b/mellea/backends/adapters/_core.py @@ -9,9 +9,18 @@ - :class:`IOContract` — ABC for prompt building and output parsing - :class:`WeightsBinding` — pluggable ABC for weights lifecycle management -Also provides :class:`LocalFileBinding`, two stub :class:`WeightsBinding` -subclasses (:class:`EmbeddedBinding`, :class:`ServerMediatedBinding`), and -:class:`AdapterSchemaMismatchError`. +Also provides: + +- :class:`LocalFileBinding` +- :class:`EmbeddedBinding` — stub :class:`WeightsBinding` subclass +- :class:`ServerMediatedBinding` — stub :class:`WeightsBinding` subclass +- :class:`AdapterSchemaMismatchError` +- :class:`_DictContract`, :class:`_ListContract` — generic, capability-agnostic + :class:`IOContract` implementations that validate required keys on a JSON + object or a JSON array of objects, respectively. Capability-*specific* + contracts (e.g. the guardian adapters' nested-key shapes) live in + :mod:`~mellea.backends.adapters.io_contracts` instead, alongside the + registry that maps every catalogued adapter function to its contract. Note: The existing :class:`~mellea.backends.adapters.adapter.Adapter` ABC in @@ -44,6 +53,11 @@ "{cls} is a Phase 0 stub; implementation lands in Epic #929 Phase 2." ) +_BUILD_PROMPT_NOT_IMPLEMENTED = ( + "build_prompt is not implemented; request construction still goes " + "through the legacy formatter/rewriter path, not IOContract." +) + class AdapterSchemaMismatchError(Exception): """Raised by :meth:`IOContract.parse` when output cannot satisfy the declared contract. @@ -52,24 +66,33 @@ class AdapterSchemaMismatchError(Exception): name (str): Name of the adapter whose contract was violated. observed_keys (frozenset[str]): Keys present in the observed output. expected_keys (frozenset[str]): Keys required by the contract. + reason (str | None): Capability-specific explanation of the mismatch. """ def __init__( - self, name: str, observed_keys: frozenset[str], expected_keys: frozenset[str] + self, + name: str, + observed_keys: frozenset[str], + expected_keys: frozenset[str], + reason: str | None = None, ) -> None: self.name = name self.observed_keys = observed_keys self.expected_keys = expected_keys - # Pass the structured fields (not the formatted message) to Exception so - # that ``self.args`` round-trips through ``pickle`` / ``copy`` — the default - # ``Exception.__reduce__`` reconstructs by calling ``cls(*self.args)``. + # Preserve the existing three-item ``args`` shape for callers and + # cross-version pickle compatibility. ``reason`` lives in instance state, + # which pickle restores after calling this constructor with ``args``. + self.reason = reason super().__init__(name, observed_keys, expected_keys) def __str__(self) -> str: - return ( + message = ( f"Adapter '{self.name}' output cannot satisfy declared contract. " f"Observed keys: {self.observed_keys}; expected: {self.expected_keys}." ) + if self.reason is not None: + message += f" Reason: {self.reason}." + return message @dataclass(frozen=True) @@ -156,9 +179,7 @@ def __init__(self, name: str, required_keys: frozenset[str]) -> None: self._required_keys = required_keys def build_prompt(self, **_kwargs: object) -> Component: - raise NotImplementedError( - "build_prompt is not used in Phase 1; implemented in Phase 2." - ) + raise NotImplementedError(_BUILD_PROMPT_NOT_IMPLEMENTED) def parse(self, raw: str) -> dict[str, object]: """Parse and validate dict-shaped adapter output. @@ -186,6 +207,63 @@ def parse(self, raw: str) -> dict[str, object]: return data +class _ListContract(IOContract): + """Validate list-of-dicts adapter output and wrap it under key `"items"`. + + Each item in the list is checked for the declared required keys. The + validated list is returned wrapped in `{"items": [...]}` so that + :func:`~mellea.stdlib.components.intrinsic._util.call_intrinsic` can always + return a plain `dict`. + + Args: + name: Adapter capability name; included in + :class:`~mellea.backends.adapters.AdapterSchemaMismatchError` messages. + required_item_keys: Keys that must be present in every item dict. + """ + + def __init__(self, name: str, required_item_keys: frozenset[str]) -> None: + self._name = name + self._required_item_keys = required_item_keys + + def build_prompt(self, **_kwargs: object) -> Component: + raise NotImplementedError(_BUILD_PROMPT_NOT_IMPLEMENTED) + + def parse(self, raw: str) -> dict[str, object]: + """Parse and validate a list-of-dicts adapter output. + + Args: + raw (str): Raw JSON string from the model. + + Returns: + dict[str, object]: `{"items": [list of validated dicts]}`. + An empty list parses to `{"items": []}`. + + Raises: + ValueError: When *raw* is not valid JSON, is not a JSON array, or + contains a non-object element. + AdapterSchemaMismatchError: When any item is missing a required key. + """ + data = json.loads(raw) + if not isinstance(data, list): + raise ValueError( + f"Adapter '{self._name}' output must be a JSON array, " + f"got {type(data).__name__}." + ) + for item in data: + if not isinstance(item, dict): + raise ValueError( + f"Adapter '{self._name}' output array must contain only JSON " + f"objects, got a {type(item).__name__} element." + ) + observed = frozenset(item.keys()) + missing = self._required_item_keys - observed + if missing: + raise AdapterSchemaMismatchError( + self._name, observed, self._required_item_keys + ) + return {"items": data} + + class WeightsBinding(abc.ABC): """Abstract lifecycle interface for adapter weights. @@ -653,10 +731,8 @@ class Adapter: # right invariant — the two feed different lookup paths (registration and the # verbs key on the binding's `qualified_name`; `_find_adapter` scans on the # identity) and both return `None` on a miss, so a disagreement surfaces as - # "adapter not found" far from its cause. But it cannot be enforced yet: the - # ten module-level `Adapter` constants in `stdlib/components/intrinsic/rag.py` - # and `guardian.py` pair an `alora` identity with a bare, deliberately - # unconfigured `LocalFileBinding()` that defaults to LoRA. Every catalogue - # entry supports both types, so those are placeholders rather than genuine - # conflicts, and the check fired on "not configured yet". Enforce it once - # #1516 gives those constants real bindings. + # "adapter not found" far from its cause. But it cannot be enforced yet: + # the deprecated shims carry a `_ShimWeightsBinding` with no `adapter_type` + # to compare at all (their identity tracks the configured type). Enforce + # the check once those constructions carry real, typed bindings (the shims + # retire in #1144). diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 0e5bb2e3f..146b04e08 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -34,11 +34,11 @@ Adapter as _AdapterCore, AdapterSchemaMismatchError, Identity, - IOContract, LocalFileBinding, WeightsBinding, ) from .catalog import AdapterType, fetch_intrinsic_metadata +from .io_contracts import get_io_contract class Adapter(abc.ABC): @@ -96,20 +96,6 @@ def get_local_hf_path(self, base_model_name: str) -> str: ... -class _ShimIOContract(IOContract): - """Phase 1 placeholder; Phase 2 (issue #1137) implements real I/O.""" - - def build_prompt(self, **kwargs: object): # type: ignore[override] - raise NotImplementedError( - "Phase 2 (issue #1137) — IOContract not yet implemented" - ) - - def parse(self, raw: str) -> dict[str, object]: - raise NotImplementedError( - "Phase 2 (issue #1137) — IOContract not yet implemented" - ) - - class _ShimWeightsBinding(WeightsBinding): """Phase 1 placeholder; Phase 2 (see epic #929) wires in real lifecycle.""" @@ -137,7 +123,7 @@ def release(self) -> None: class IntrinsicAdapter(LocalHFAdapter, _AdapterCore): """Deprecated shim for adapters that implement adapter functions. - .. deprecated:: + Deprecated: Use :class:`~mellea.backends.adapters.Adapter` directly. `IntrinsicAdapter` will be removed in a future release (Epic #929, issue #1144). @@ -171,12 +157,13 @@ class IntrinsicAdapter(LocalHFAdapter, _AdapterCore): adapter_type (AdapterType): The adapter type (`LORA` or `ALORA`). config (dict): Parsed I/O transformation configuration for the adapter function. - .. note:: - `identity`, `io_contract`, and `weights` are Phase 1 internal scaffolding - populated in `__init__` to satisfy the new :class:`~mellea.backends.adapters.Adapter` - protocol. They are not meaningful consumer-facing attributes; `io_contract` and - `weights` raise :exc:`NotImplementedError` and will be replaced in Phase 2 - (issues #1137, #1141). + Note: + `identity`, `io_contract`, and `weights` are internal scaffolding populated + in `__init__` to satisfy the :class:`~mellea.backends.adapters.Adapter` + protocol; they are not meaningful consumer-facing attributes. `io_contract` + is the real, declared contract for `intrinsic_name` (issue #1516); `weights` + remains the Phase 1 `_ShimWeightsBinding` placeholder and raises + `NotImplementedError` until Phase 2 (issue #1141) replaces it. """ def __setattr__(self, name: str, value: object) -> None: @@ -256,6 +243,9 @@ def __init__( self.config: dict = config_dict # Populate the new Adapter triple so isinstance(self, _AdapterCore) holds. + # io_contract comes from the same registry resolve_adapter() consults + # (see issue #1516), not a placeholder. weights stays the Phase 2 + # _ShimWeightsBinding placeholder; that axis is #1141/#1142. _AdapterCore.__init__( self, identity=Identity( @@ -265,7 +255,7 @@ def __init__( else "lora", capability=intrinsic_name, ), - io_contract=_ShimIOContract(), + io_contract=get_io_contract(intrinsic_name), weights=_ShimWeightsBinding(), ) @@ -891,7 +881,7 @@ def _find_adapter( class EmbeddedIntrinsicAdapter(_AdapterCore): """Deprecated shim for adapter functions embedded in a Granite Switch model. - .. deprecated:: + Deprecated: Use :class:`~mellea.backends.adapters.Adapter` directly. `EmbeddedIntrinsicAdapter` will be removed in a future release (Epic #929, issue #1144). @@ -915,12 +905,18 @@ class EmbeddedIntrinsicAdapter(_AdapterCore): config (dict): Parsed I/O transformation configuration. technology (str): `"lora"` or `"alora"`. - .. note:: - `identity`, `io_contract`, and `weights` are Phase 1 internal scaffolding - populated in `__init__` to satisfy the new :class:`~mellea.backends.adapters.Adapter` - protocol. They are not meaningful consumer-facing attributes; `io_contract` and - `weights` raise :exc:`NotImplementedError` and will be replaced in Phase 2 - (issues #1137, #1142). + Note: + `identity`, `io_contract`, and `weights` are internal scaffolding + populated in `__init__` to satisfy the `Adapter` protocol; they are + not meaningful consumer-facing attributes. + + - `identity`: always a real value. + + - `io_contract`: the real, declared contract for `intrinsic_name` + (issue #1516); no longer a placeholder. + + - `weights`: Phase 1 `_ShimWeightsBinding` placeholder; raises + `NotImplementedError` until issue #1142 replaces it. """ def __setattr__(self, name: str, value: object) -> None: @@ -958,15 +954,18 @@ def __init__(self, intrinsic_name: str, config: dict, technology: str = "lora"): # Populate the new Adapter triple so isinstance(self, _AdapterCore) holds. # technology is validated above; cast to the Literal type mypy expects. + identity = Identity( + name=intrinsic_name, + adapter_type=cast(Literal["lora", "alora"], technology), + capability=intrinsic_name, + ) + + io_contract = get_io_contract(intrinsic_name) + + weights = _ShimWeightsBinding() + _AdapterCore.__init__( - self, - identity=Identity( - name=intrinsic_name, - adapter_type=cast(Literal["lora", "alora"], technology), - capability=intrinsic_name, - ), - io_contract=_ShimIOContract(), - weights=_ShimWeightsBinding(), + self, identity=identity, io_contract=io_contract, weights=weights ) @staticmethod diff --git a/mellea/backends/adapters/io_contracts.py b/mellea/backends/adapters/io_contracts.py new file mode 100644 index 000000000..91ff2c59c --- /dev/null +++ b/mellea/backends/adapters/io_contracts.py @@ -0,0 +1,288 @@ +# Copyright IBM Corp. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Canonical output contracts for adapter functions (Epic #929 Phase 2, issue #1516). + +Before this module existed, an adapter function's output contract travelled to +:func:`~mellea.stdlib.components.intrinsic._util.call_intrinsic` as a caller-supplied +argument, built from a module-level constant in `rag.py` or `guardian.py`, while the +adapter actually resolved by :meth:`~mellea.backends.adapters.AdapterMixin.resolve_adapter` +carried an unrelated placeholder contract. Nothing tied the two together, so passing the +wrong constant was a silent mismatch. + +:data:`_INTRINSIC_IO_CONTRACTS` is the single source of truth instead: it is keyed by the +adapter function's catalog name (:attr:`~mellea.backends.adapters.catalog.IntrinsicsCatalogEntry.name`, +e.g. `"guardian-core"` — the same string passed to `call_intrinsic` and +`resolve_adapter`). The shim adapters in `adapter.py` read from it at construction +(via :func:`get_io_contract`); the high-level helpers reach it indirectly, through +the adapter `resolve_adapter` returns. Declaring a capability's contract anywhere +else reintroduces the parallel-argument problem this module exists to close. + +An adapter a user registers with a backend takes precedence over the registry for its +own contract: `call_intrinsic` parses with whatever contract the resolved adapter +carries, and only adapters Mellea constructs itself (the lazy shims) get theirs from here. +""" + +import json +import math + +from ...core import Component +from ._core import ( + _BUILD_PROMPT_NOT_IMPLEMENTED, + AdapterSchemaMismatchError, + IOContract, + _DictContract, + _ListContract, +) +from .catalog import known_intrinsic_names + + +class _PolicyGuardrailsContract(IOContract): + """Validate policy-guardrails adapter output: exactly one of `label` or `score`. + + The adapter returns either `{"label": "Yes"|"No"|"Ambiguous"}` or + `{"score": "Yes"|"No"|"Ambiguous"}` — never both, never neither. + """ + + def build_prompt(self, **_kwargs: object) -> Component: + raise NotImplementedError(_BUILD_PROMPT_NOT_IMPLEMENTED) + + def parse(self, raw: str) -> dict[str, object]: + """Parse and validate policy-guardrails output. + + Args: + raw (str): Raw JSON string from the model. + + Returns: + dict[str, object]: Parsed output dict with exactly one of `label` or `score`. + + Raises: + ValueError: When *raw* is not valid JSON or is not a JSON object. + AdapterSchemaMismatchError: When neither or both of `label` / `score` are present. + """ + data = json.loads(raw) + if not isinstance(data, dict): + raise ValueError( + f"Adapter 'policy-guardrails' output must be a JSON object, " + f"got {type(data).__name__}." + ) + has_label = "label" in data + has_score = "score" in data + if not has_label and not has_score: + raise AdapterSchemaMismatchError( + "policy-guardrails", + frozenset(data.keys()), + frozenset({"label", "score"}), + "neither `label` nor `score` was present", + ) + if has_label and has_score: + raise AdapterSchemaMismatchError( + "policy-guardrails", + frozenset(data.keys()), + frozenset({"label", "score"}), + "both `label` and `score` were present", + ) + return data + + +class _GuardianCheckContract(IOContract): + """Validate guardian-core adapter output: `{"guardian": {"score": }}`. + + Checks that the outer `guardian` key is present and that it contains + a nested `score` key. + """ + + def build_prompt(self, **_kwargs: object) -> Component: + raise NotImplementedError(_BUILD_PROMPT_NOT_IMPLEMENTED) + + def parse(self, raw: str) -> dict[str, object]: + """Parse and validate guardian-core output. + + Args: + raw (str): Raw JSON string from the model. + + Returns: + dict[str, object]: Parsed output dict containing `{"guardian": {"score": ...}}`. + + Raises: + ValueError: When *raw* is not valid JSON or is not a JSON object. + AdapterSchemaMismatchError: When `guardian` key is absent or `guardian.score` + is absent. + """ + data = json.loads(raw) + if not isinstance(data, dict): + raise ValueError( + f"Adapter 'guardian-core' output must be a JSON object, " + f"got {type(data).__name__}." + ) + if "guardian" not in data: + raise AdapterSchemaMismatchError( + "guardian-core", frozenset(data.keys()), frozenset({"guardian"}) + ) + guardian_val = data["guardian"] + if not isinstance(guardian_val, dict) or "score" not in guardian_val: + raise AdapterSchemaMismatchError( + "guardian-core", + frozenset(guardian_val.keys()) + if isinstance(guardian_val, dict) + else frozenset(data.keys()), + frozenset({"score"}), + ) + return data + + +class _RequirementCheckContract(IOContract): + """Validate requirement-check output: `{"requirement_check": {"score": <0.0-1.0 float>}}`. + + Consolidates the score-range validation that both production consumers of this + capability's output — `core.requirement_check()` + (`mellea/stdlib/components/intrinsic/core.py`) and `requirement_check_to_bool()` + (`mellea/stdlib/requirements/requirement.py`) — used to hand-roll independently, + into one declared contract, per issue #1516. + """ + + def build_prompt(self, **_kwargs: object) -> Component: + raise NotImplementedError(_BUILD_PROMPT_NOT_IMPLEMENTED) + + def parse(self, raw: str) -> dict[str, object]: + """Parse and validate requirement-check output. + + Args: + raw (str): Raw JSON string from the model. + + Returns: + dict[str, object]: Parsed output dict containing + `{"requirement_check": {"score": }}`. + + Raises: + ValueError: When *raw* is not valid JSON or is not a JSON object. + AdapterSchemaMismatchError: When `requirement_check` is absent or not a + dict, or when its `score` is absent, not a finite number, is a `bool`, + or falls outside the closed range `[0.0, 1.0]`. + """ + data = json.loads(raw) + if not isinstance(data, dict): + raise ValueError( + f"Adapter 'requirement-check' output must be a JSON object, " + f"got {type(data).__name__}." + ) + req_check = data.get("requirement_check") + if not isinstance(req_check, dict): + raise AdapterSchemaMismatchError( + "requirement-check", + frozenset(data.keys()), + frozenset({"requirement_check"}), + ) + score = req_check.get("score") + if ( + not isinstance(score, (int, float)) + or isinstance(score, bool) # bool subclasses int; exclude it explicitly + or not math.isfinite(score) + or not 0.0 <= score <= 1.0 + ): + raise AdapterSchemaMismatchError( + "requirement-check", frozenset(req_check.keys()), frozenset({"score"}) + ) + return data + + +_INTRINSIC_IO_CONTRACTS: dict[str, IOContract] = { + "answerability": _DictContract("answerability", frozenset({"answerability"})), + "query_rewrite": _DictContract("query_rewrite", frozenset({"rewritten_question"})), + "query_clarification": _DictContract( + "query_clarification", frozenset({"clarification"}) + ), + "citations": _ListContract( + "citations", + frozenset( + { + "response_begin", + "response_end", + "response_text", + "citation_doc_id", + "citation_begin", + "citation_end", + "citation_text", + } + ), + ), + "context_relevance": _DictContract( + "context_relevance", frozenset({"context_relevance"}) + ), + "hallucination_detection": _ListContract( + "hallucination_detection", + frozenset( + { + "response_begin", + "response_end", + "response_text", + "faithfulness", + "explanation", + } + ), + ), + "policy-guardrails": _PolicyGuardrailsContract(), + "guardian-core": _GuardianCheckContract(), + "factuality-detection": _DictContract("factuality-detection", frozenset({"score"})), + "factuality-correction": _DictContract( + "factuality-correction", frozenset({"correction"}) + ), + "uncertainty": _DictContract("uncertainty", frozenset({"certainty"})), + "requirement-check": _RequirementCheckContract(), + "context-attribution": _ListContract( + "context-attribution", + frozenset( + { + "response_begin", + "response_end", + "response_text", + "attribution_doc_id", + "attribution_msg_index", + "attribution_begin", + "attribution_end", + "attribution_text", + } + ), + ), +} +# Canonical output contract for every built-in adapter function, keyed by its +# catalog `name` (see module docstring). Capture the built-in names before +# `CustomIntrinsicAdapter` can extend the global catalogue at runtime, then keep +# the registry exhaustive over that fixed set at import time — mirroring the +# duplicate-`effective_capability` check in catalog.py. +_BUILTIN_INTRINSIC_NAMES = frozenset(known_intrinsic_names()) +_missing_contracts = _BUILTIN_INTRINSIC_NAMES - set(_INTRINSIC_IO_CONTRACTS) +if _missing_contracts: + raise ValueError( + f"Built-in catalogued adapter functions with no declared IOContract: " + f"{_missing_contracts}" + ) +del _missing_contracts + + +def get_io_contract(name: str) -> IOContract: + """Return the canonical output contract for an adapter function. + + Args: + name (str): Catalog name of the adapter function (e.g. `"answerability"`, + `"guardian-core"`) — :attr:`~mellea.backends.adapters.catalog.IntrinsicsCatalogEntry.name`, + never :attr:`~mellea.backends.adapters.catalog.IntrinsicsCatalogEntry.effective_capability`. + This is the same string passed to + :func:`~mellea.stdlib.components.intrinsic._util.call_intrinsic` and + :meth:`~mellea.backends.adapters.AdapterMixin.resolve_adapter`. + + Returns: + IOContract: The contract declared in :data:`_INTRINSIC_IO_CONTRACTS` for + `name`. Adapter functions outside the catalog (e.g. one registered + through the deprecated `CustomIntrinsicAdapter`) fall back to a dict + contract with no required keys — permissive about *which* keys are + present, but still requires the parsed JSON to be an object; a + top-level array or scalar still raises `ValueError`, since Mellea has + no declared schema for these adapter functions to relax that far. + """ + contract = _INTRINSIC_IO_CONTRACTS.get(name) + if contract is not None: + return contract + # Custom/user-defined adapters outside the catalog have no declared schema; + # accept any JSON object rather than rejecting on an arbitrary key set. + return _DictContract(name, frozenset()) diff --git a/mellea/stdlib/components/intrinsic/_util.py b/mellea/stdlib/components/intrinsic/_util.py index 3d939a560..0b1034954 100644 --- a/mellea/stdlib/components/intrinsic/_util.py +++ b/mellea/stdlib/components/intrinsic/_util.py @@ -3,12 +3,11 @@ """Shared utilities for intrinsic convenience wrappers.""" -import json from typing import cast from ....backends import ModelOption from ....backends._options import resolve_model_options -from ....backends.adapters import AdapterMixin, AdapterType, IOContract +from ....backends.adapters import AdapterMixin, AdapterType from ....core import Backend from ....stdlib import functional as mfuncs from ...components import Document @@ -176,18 +175,19 @@ def call_intrinsic( /, kwargs: dict | None = None, model_options: dict | None = None, - io_contract: IOContract | None = None, ) -> dict[str, object]: - """Invoke an adapter function via the backend, returning parsed and optionally validated JSON output. - - Uses `AdapterMixin.resolve_adapter` to find or lazily register the adapter, - then executes via `mfuncs.act`. - - When *io_contract* is provided its :meth:`~mellea.backends.adapters.IOContract.parse` - method is called on the raw output string before returning. The contract validates - required fields and raises :class:`~mellea.backends.adapters.AdapterSchemaMismatchError` - on contract-breaking deltas; forward-compatible additions (extra optional fields) do - not raise. When *io_contract* is `None` the raw `json.loads` result is returned. + """Invoke an adapter function via the backend, returning parsed and validated JSON output. + + Uses `AdapterMixin.resolve_adapter` to find or lazily register the adapter, then + executes via `mfuncs.act`. The resolved adapter's own + `IOContract.parse` method is called on the raw output string before returning — the + output contract always travels with the adapter that produced it, rather than as a + separate argument a caller could mismatch. For adapters Mellea constructs (the lazy + shims) that contract comes from the `mellea.backends.adapters.io_contracts` registry; + for an adapter a user registered under the same capability, the user's own + `io_contract` wins. The contract validates required fields and + raises `AdapterSchemaMismatchError` on contract-breaking deltas; forward-compatible + additions (extra optional fields) do not raise. Args: intrinsic_name (str): Capability name of the adapter function @@ -199,23 +199,25 @@ def call_intrinsic( model_options (dict | None): Model options that override defaults. Adapter functions default to `TEMPERATURE: 0.0` for deterministic output; pass `TEMPERATURE` here to override it. - io_contract (IOContract | None): Output contract to validate and parse the - raw model output. When `None`, `json.loads` is used directly. Returns: - dict[str, object]: Parsed (and optionally validated) JSON output from the adapter function. + dict[str, object]: Parsed and validated JSON output from the adapter function. Raises: ValueError: When *context* forwards no history to the model (e.g. a - `SimpleContext` was passed), or when the model output is `None` or - is not valid JSON. - AdapterSchemaMismatchError: When *io_contract* is provided and the - model output is missing a required field. + `SimpleContext` was passed), when the model output is `None` or is + not valid JSON, or when well-formed JSON is rejected for shape by + the resolved adapter's contract (wrong top-level type, or a + non-object element of an array-shaped contract). + AdapterSchemaMismatchError: When the model output is missing a field required + by the resolved adapter's output contract. """ _assert_context_forwards_history(intrinsic_name, context) - # Ensure the adapter is registered; resolve_adapter creates it if absent. - backend.resolve_adapter(intrinsic_name) + # Resolve (finding or lazily registering) the adapter now, rather than merely + # ensuring it is registered and discarding the result: its io_contract is what + # parses the raw output below. + adapter = backend.resolve_adapter(intrinsic_name) # Adapter activation is the backend's responsibility — the HF backend acquires # its generation lock and sets the active adapter inside _generate_with_adapter_lock, @@ -247,6 +249,4 @@ def call_intrinsic( result_str = model_output_thunk.value if result_str is None: raise ValueError("Model output is None.") - if io_contract is not None: - return io_contract.parse(result_str) - return json.loads(result_str) + return adapter.io_contract.parse(result_str) diff --git a/mellea/stdlib/components/intrinsic/core.py b/mellea/stdlib/components/intrinsic/core.py index e2feeddad..bd75f4a3e 100644 --- a/mellea/stdlib/components/intrinsic/core.py +++ b/mellea/stdlib/components/intrinsic/core.py @@ -4,10 +4,9 @@ """Adapter functions for core model capabilities.""" import collections.abc -import math from typing import cast -from ....backends.adapters import AdapterMixin, AdapterSchemaMismatchError +from ....backends.adapters import AdapterMixin from ...components import Document, Message from ...context import ChatContext from ..docs.document import _coerce_to_documents @@ -23,6 +22,9 @@ def check_certainty( assistant's response to a user's question. The context should end with a user question followed by an assistant answer. + Output contract — required key: `certainty`. Missing the key raises + `AdapterSchemaMismatchError`; extra optional keys do not raise (forward-compatible). + Args: context: Chat context containing user question and assistant answer. backend: Backend instance that supports LoRA/aLoRA adapters. @@ -32,6 +34,12 @@ def check_certainty( Returns: Certainty score as a float (higher = more certain). + + Raises: + ValueError: When the model output is not valid JSON or is not a + JSON object. + AdapterSchemaMismatchError: When the model output is missing the required + `certainty` field. """ result_json = call_intrinsic( "uncertainty", context, backend, model_options=model_options @@ -52,6 +60,10 @@ def requirement_check( `io.yaml` `instruction` template via `IntrinsicsRewriter`, which appends the formatted evaluation prompt as a new user message. + Output contract — required shape: `{"requirement_check": {"score": }}`, + with `score` a finite number (not a `bool`) in the closed range `[0.0, 1.0]`. + Any deviation raises `AdapterSchemaMismatchError`. + Args: context: Chat context containing user question and assistant answer. backend: Backend instance that supports LoRA/aLoRA adapters. @@ -64,6 +76,8 @@ def requirement_check( Score as a float between 0.0 and 1.0 (higher = more likely satisfied). Raises: + ValueError: When the model output is not valid JSON or is not a + JSON object. AdapterSchemaMismatchError: If the adapter output does not match the expected `{"requirement_check": {"score": }}` contract, or if the score is not a finite number in the range 0.0-1.0. @@ -75,27 +89,9 @@ def requirement_check( kwargs={"requirement": requirement}, model_options=model_options, ) - # Mirrors the validation in requirement_check_to_bool() in requirement.py; Phase 2 will consolidate via IOContract. - req_check = result_json.get("requirement_check") - if not isinstance(req_check, dict): - raise AdapterSchemaMismatchError( - name="requirement-check", - observed_keys=frozenset(result_json.keys()), - expected_keys=frozenset({"requirement_check"}), - ) - score = req_check.get("score") - if ( - not isinstance(score, (int, float)) - or isinstance(score, bool) # bool subclasses int; exclude it explicitly - or not math.isfinite(score) - or not 0.0 <= score <= 1.0 - ): - raise AdapterSchemaMismatchError( - name="requirement-check", - observed_keys=frozenset(req_check.keys()), - expected_keys=frozenset({"score"}), - ) - return score + return cast( + float, cast(dict[str, object], result_json["requirement_check"])["score"] + ) def find_context_attributions( @@ -111,6 +107,12 @@ def find_context_attributions( documents that were most important to the LLM in generating each sentence in the assistant response. + Output contract — each record must contain: `response_begin`, `response_end`, + `response_text`, `attribution_doc_id`, `attribution_msg_index`, + `attribution_begin`, `attribution_end`, `attribution_text`. A record missing + any of these keys raises `AdapterSchemaMismatchError`; extra optional keys do + not raise (forward-compatible). + Args: response (str | None): Assistant response. When `None`, extracted from the last assistant output in `context`. @@ -133,6 +135,12 @@ def find_context_attributions( `attribution_begin`, `attribution_end`, and `attribution_text`. Begin and end offsets are character offsets into their respective UTF-8 strings. + + Raises: + ValueError: When the model output is not valid JSON, is not a + JSON array, or contains a non-object element. + AdapterSchemaMismatchError: When any record in the output is missing a + required field. """ response, context = _resolve_response(response, context) result_json = call_intrinsic( @@ -147,4 +155,4 @@ def find_context_attributions( backend, model_options=model_options, ) - return cast(list[dict], result_json) + return cast(list[dict], result_json["items"]) diff --git a/mellea/stdlib/components/intrinsic/guardian.py b/mellea/stdlib/components/intrinsic/guardian.py index 8730e9496..982233604 100644 --- a/mellea/stdlib/components/intrinsic/guardian.py +++ b/mellea/stdlib/components/intrinsic/guardian.py @@ -13,20 +13,10 @@ """ import collections.abc -import json import warnings from typing import cast -from ....backends.adapters import ( - Adapter, - AdapterMixin, - AdapterSchemaMismatchError, - Identity, - IOContract, - LocalFileBinding, -) -from ....backends.adapters._core import _DictContract -from ....core import Component +from ....backends.adapters import AdapterMixin from ....core.utils import MelleaLogger from ...components import Document from ...context import ChatContext @@ -42,140 +32,6 @@ """Mapping used by the deprecated `target_role` path of `guardian_check`.""" -# --------------------------------------------------------------------------- -# IOContract implementations -# --------------------------------------------------------------------------- - - -class _PolicyGuardrailsContract(IOContract): - """Validate policy-guardrails adapter output: exactly one of `label` or `score`. - - The adapter returns either `{"label": "Yes"|"No"|"Ambiguous"}` or - `{"score": "Yes"|"No"|"Ambiguous"}` — never both, never neither. - """ - - def build_prompt(self, **_kwargs: object) -> Component: - raise NotImplementedError( - "build_prompt is not used in Phase 1; implemented in Phase 2." - ) - - def parse(self, raw: str) -> dict[str, object]: - """Parse and validate policy-guardrails output. - - Args: - raw (str): Raw JSON string from the model. - - Returns: - dict[str, object]: Parsed output dict with exactly one of `label` or `score`. - - Raises: - ValueError: When *raw* is not valid JSON or is not a JSON object. - AdapterSchemaMismatchError: When neither or both of `label` / `score` are present. - """ - data = json.loads(raw) - if not isinstance(data, dict): - raise ValueError( - f"Adapter 'policy-guardrails' output must be a JSON object, " - f"got {type(data).__name__}." - ) - has_label = "label" in data - has_score = "score" in data - if not has_label and not has_score: - raise AdapterSchemaMismatchError( - "policy-guardrails", - frozenset(data.keys()), - frozenset({"label", "score"}), - ) - if has_label and has_score: - raise AdapterSchemaMismatchError( - "policy-guardrails", - frozenset(data.keys()), - frozenset({"label", "score"}), - ) - return data - - -class _GuardianCheckContract(IOContract): - """Validate guardian-core adapter output: `{"guardian": {"score": }}`. - - Checks that the outer `guardian` key is present and that it contains - a nested `score` key. - """ - - def build_prompt(self, **_kwargs: object) -> Component: - raise NotImplementedError( - "build_prompt is not used in Phase 1; implemented in Phase 2." - ) - - def parse(self, raw: str) -> dict[str, object]: - """Parse and validate guardian-core output. - - Args: - raw (str): Raw JSON string from the model. - - Returns: - dict[str, object]: Parsed output dict containing `{"guardian": {"score": ...}}`. - - Raises: - ValueError: When *raw* is not valid JSON or is not a JSON object. - AdapterSchemaMismatchError: When `guardian` key is absent or `guardian.score` - is absent. - """ - data = json.loads(raw) - if not isinstance(data, dict): - raise ValueError( - f"Adapter 'guardian-core' output must be a JSON object, " - f"got {type(data).__name__}." - ) - if "guardian" not in data: - raise AdapterSchemaMismatchError( - "guardian-core", frozenset(data.keys()), frozenset({"guardian"}) - ) - guardian_val = data["guardian"] - if not isinstance(guardian_val, dict) or "score" not in guardian_val: - raise AdapterSchemaMismatchError( - "guardian-core", - frozenset(guardian_val.keys()) - if isinstance(guardian_val, dict) - else frozenset(data.keys()), - frozenset({"score"}), - ) - return data - - -# --------------------------------------------------------------------------- -# Module-level Adapter constants (one per helper) -# --------------------------------------------------------------------------- - -_POLICY_GUARDRAILS_ADAPTER = Adapter( - identity=Identity("policy-guardrails", "alora", capability="policy_guardrails"), - io_contract=_PolicyGuardrailsContract(), - weights=LocalFileBinding(), -) - -_GUARDIAN_CHECK_ADAPTER = Adapter( - identity=Identity("guardian-core", "alora", capability="guardian_core"), - io_contract=_GuardianCheckContract(), - weights=LocalFileBinding(), -) - -_FACTUALITY_DETECTION_ADAPTER = Adapter( - identity=Identity( - "factuality-detection", "alora", capability="factuality_detection" - ), - io_contract=_DictContract("factuality-detection", frozenset({"score"})), - weights=LocalFileBinding(), -) - -_FACTUALITY_CORRECTION_ADAPTER = Adapter( - identity=Identity( - "factuality-correction", "alora", capability="factuality_correction" - ), - io_contract=_DictContract("factuality-correction", frozenset({"correction"})), - weights=LocalFileBinding(), -) - - def policy_guardrails( context: ChatContext, backend: AdapterMixin, @@ -203,7 +59,8 @@ def policy_guardrails( Compliance label as `"Yes"`, `"No"`, or `"Ambiguous"` (`"Yes"` = compliant). Raises: - ValueError: When the model output is not valid JSON. + ValueError: When the model output is not valid JSON or is not a + JSON object. AdapterSchemaMismatchError: When neither or both of `label` / `score` are present. """ result_json = call_intrinsic( @@ -212,7 +69,6 @@ def policy_guardrails( backend, kwargs={"policy_text": policy_text}, model_options=model_options, - io_contract=_POLICY_GUARDRAILS_ADAPTER.io_contract, ) if "label" in result_json: @@ -366,7 +222,7 @@ def guardian_check( Raises: TypeError: When both `scoring_schema` and `target_role` are provided. ValueError: When `target_role` is not `"user"` or `"assistant"`, or - when the model output is not valid JSON. + when the model output is not valid JSON or is not a JSON object. AdapterSchemaMismatchError: When the model output is missing the required `guardian` key or its nested `score` key. """ @@ -412,7 +268,6 @@ def guardian_check( backend, kwargs={"criteria": criteria_text, "scoring_schema": scoring_schema_text}, model_options=model_options, - io_contract=_GUARDIAN_CHECK_ADAPTER.io_contract, ) return cast(float, cast(dict[str, object], result_json["guardian"])["score"]) @@ -462,18 +317,15 @@ def factuality_detection( Raises: ValueError: If `documents` is provided but the last assistant response cannot be extracted (empty context, non-assistant last turn, or - uncomputed response). Also raised when the model output is not valid JSON. + uncomputed response). Also raised when the model output is not valid + JSON, or is not a JSON object. AdapterSchemaMismatchError: When the model output is missing the required `score` field. """ if documents is not None: context = _inject_documents(context, documents) result_json = call_intrinsic( - "factuality-detection", - context, - backend, - model_options=model_options, - io_contract=_FACTUALITY_DETECTION_ADAPTER.io_contract, + "factuality-detection", context, backend, model_options=model_options ) return cast(str, result_json["score"]) @@ -523,18 +375,15 @@ def factuality_correction( Raises: ValueError: If `documents` is provided but the last assistant response cannot be extracted (empty context, non-assistant last turn, or - uncomputed response). Also raised when the model output is not valid JSON. + uncomputed response). Also raised when the model output is not valid + JSON, or is not a JSON object. AdapterSchemaMismatchError: When the model output is missing the required `correction` field. """ if documents is not None: context = _inject_documents(context, documents) result_json = call_intrinsic( - "factuality-correction", - context, - backend, - model_options=model_options, - io_contract=_FACTUALITY_CORRECTION_ADAPTER.io_contract, + "factuality-correction", context, backend, model_options=model_options ) return cast(str, result_json["correction"]) diff --git a/mellea/stdlib/components/intrinsic/rag.py b/mellea/stdlib/components/intrinsic/rag.py index 92fde76da..0232480e5 100644 --- a/mellea/stdlib/components/intrinsic/rag.py +++ b/mellea/stdlib/components/intrinsic/rag.py @@ -4,156 +4,16 @@ """Adapter functions related to retrieval-augmented generation.""" import collections.abc -import json import warnings from typing import cast -from ....backends.adapters import ( - Adapter, - AdapterMixin, - AdapterSchemaMismatchError, - Identity, - IOContract, - LocalFileBinding, -) -from ....backends.adapters._core import _DictContract -from ....core import Component +from ....backends.adapters import AdapterMixin from ...components import Document from ...context import ChatContext from ..chat import Message from ..docs.document import _coerce_to_document, _coerce_to_documents from ._util import _resolve_question, _resolve_response, call_intrinsic -# --------------------------------------------------------------------------- -# IOContract implementations -# --------------------------------------------------------------------------- - - -class _ListContract(IOContract): - """Validate list-of-dicts adapter output and wrap it under key `"items"`. - - Each item in the list is checked for the declared required keys. The - validated list is returned wrapped in `{"items": [...]}` so that - :func:`call_intrinsic` can always return a plain `dict`. - - Args: - name: Adapter capability name; included in - :class:`~mellea.backends.adapters.AdapterSchemaMismatchError` messages. - required_item_keys: Keys that must be present in every item dict. - """ - - def __init__(self, name: str, required_item_keys: frozenset[str]) -> None: - self._name = name - self._required_item_keys = required_item_keys - - def build_prompt(self, **_kwargs: object) -> Component: - raise NotImplementedError( - "build_prompt is not used in Phase 1; implemented in Phase 2." - ) - - def parse(self, raw: str) -> dict[str, object]: - """Parse and validate a list-of-dicts adapter output. - - Args: - raw (str): Raw JSON string from the model. - - Returns: - dict[str, object]: `{"items": [list of validated dicts]}`. - An empty list parses to `{"items": []}`. - - Raises: - ValueError: When *raw* is not valid JSON, is not a JSON array, or - contains a non-object element. - AdapterSchemaMismatchError: When any item is missing a required key. - """ - data = json.loads(raw) - if not isinstance(data, list): - raise ValueError( - f"Adapter '{self._name}' output must be a JSON array, " - f"got {type(data).__name__}." - ) - for item in data: - if not isinstance(item, dict): - raise ValueError( - f"Adapter '{self._name}' output array must contain only JSON " - f"objects, got a {type(item).__name__} element." - ) - observed = frozenset(item.keys()) - missing = self._required_item_keys - observed - if missing: - raise AdapterSchemaMismatchError( - self._name, observed, self._required_item_keys - ) - return {"items": data} - - -# --------------------------------------------------------------------------- -# Module-level Adapter constants (one per helper) -# --------------------------------------------------------------------------- - -_ANSWERABILITY_ADAPTER = Adapter( - identity=Identity("answerability", "alora", capability="answerability"), - io_contract=_DictContract("answerability", frozenset({"answerability"})), - weights=LocalFileBinding(), -) - -_QUERY_REWRITE_ADAPTER = Adapter( - identity=Identity("query_rewrite", "alora", capability="query_rewrite"), - io_contract=_DictContract("query_rewrite", frozenset({"rewritten_question"})), - weights=LocalFileBinding(), -) - -_QUERY_CLARIFY_ADAPTER = Adapter( - identity=Identity("query_clarification", "alora", capability="query_clarification"), - io_contract=_DictContract("query_clarification", frozenset({"clarification"})), - weights=LocalFileBinding(), -) - -_CITATIONS_ADAPTER = Adapter( - identity=Identity("citations", "alora", capability="citations"), - io_contract=_ListContract( - "citations", - frozenset( - { - "response_begin", - "response_end", - "response_text", - "citation_doc_id", - "citation_begin", - "citation_end", - "citation_text", - } - ), - ), - weights=LocalFileBinding(), -) - -_CONTEXT_RELEVANCE_ADAPTER = Adapter( - identity=Identity("context_relevance", "alora", capability="context_relevance"), - io_contract=_DictContract("context_relevance", frozenset({"context_relevance"})), - weights=LocalFileBinding(), -) - -_HALLUCINATION_ADAPTER = Adapter( - identity=Identity( - "hallucination_detection", "alora", capability="hallucination_detection" - ), - io_contract=_ListContract( - "hallucination_detection", - frozenset( - { - "response_begin", - "response_end", - "response_text", - "faithfulness", - "explanation", - } - ), - ), - weights=LocalFileBinding(), -) - - # --------------------------------------------------------------------------- # High-level helper functions # --------------------------------------------------------------------------- @@ -194,7 +54,8 @@ def check_answerability( A string value of either `"answerable"` or `"unanswerable"`. Raises: - ValueError: When the model output is not valid JSON. + ValueError: When the model output is not valid JSON or is not a + JSON object. AdapterSchemaMismatchError: When the model output is missing the required `answerability` field. """ @@ -205,7 +66,6 @@ def check_answerability( Message("user", question, documents=_coerce_to_documents(documents)) ), backend, - io_contract=_ANSWERABILITY_ADAPTER.io_contract, model_options=model_options, ) return cast(str, result["answerability"]) @@ -241,7 +101,8 @@ def rewrite_question( Rewritten version of `question`. Raises: - ValueError: When the model output is not valid JSON. + ValueError: When the model output is not valid JSON or is not a + JSON object. AdapterSchemaMismatchError: When the model output is missing the required `rewritten_question` field. """ @@ -250,7 +111,6 @@ def rewrite_question( "query_rewrite", context.add(Message("user", question)), backend, - io_contract=_QUERY_REWRITE_ADAPTER.io_contract, model_options=model_options, ) return cast(str, result["rewritten_question"]) @@ -292,7 +152,8 @@ def clarify_query( the string `"CLEAR"` if no clarification is needed. Raises: - ValueError: When the model output is not valid JSON. + ValueError: When the model output is not valid JSON or is not a + JSON object. AdapterSchemaMismatchError: When the model output is missing the required `clarification` field. """ @@ -303,7 +164,6 @@ def clarify_query( Message("user", question, documents=_coerce_to_documents(documents)) ), backend, - io_contract=_QUERY_CLARIFY_ADAPTER.io_contract, model_options=model_options, ) return cast(str, result["clarification"]) @@ -352,7 +212,8 @@ def find_citations( character offsets into their respective UTF-8 strings. Raises: - ValueError: When the model output is not valid JSON. + ValueError: When the model output is not valid JSON, is not a + JSON array, or contains a non-object element. AdapterSchemaMismatchError: When any record in the output is missing a required field. """ @@ -367,7 +228,6 @@ def find_citations( ) ), backend, - io_contract=_CITATIONS_ADAPTER.io_contract, model_options=model_options, ) return cast(list[dict], result["items"]) @@ -417,7 +277,8 @@ def check_context_relevance( `"relevant"`, `"irrelevant"`, or `"partially relevant"`. Raises: - ValueError: When the model output is not valid JSON. + ValueError: When the model output is not valid JSON or is not a + JSON object. AdapterSchemaMismatchError: When the model output is missing the required `context_relevance` field. """ @@ -436,7 +297,6 @@ def check_context_relevance( context.add(Message("user", question)), backend, kwargs={"document_content": document.text}, - io_contract=_CONTEXT_RELEVANCE_ADAPTER.io_contract, model_options=model_options, ) return cast(str, result["context_relevance"]) @@ -485,7 +345,8 @@ def flag_hallucinated_content( `response_text`, `faithfulness`, `explanation`. Raises: - ValueError: When the model output is not valid JSON. + ValueError: When the model output is not valid JSON, is not a + JSON array, or contains a non-object element. AdapterSchemaMismatchError: When any record in the output is missing a required field. """ @@ -496,7 +357,6 @@ def flag_hallucinated_content( Message("assistant", response, documents=_coerce_to_documents(documents)) ), backend, - io_contract=_HALLUCINATION_ADAPTER.io_contract, model_options=model_options, ) return cast(list[dict], result["items"]) diff --git a/mellea/stdlib/requirements/requirement.py b/mellea/stdlib/requirements/requirement.py index e102880f9..fc072f2ae 100644 --- a/mellea/stdlib/requirements/requirement.py +++ b/mellea/stdlib/requirements/requirement.py @@ -3,12 +3,10 @@ """Requirements are a special type of Component used as input to the "validate" step in Instruct/Validate/Repair design patterns.""" -import json -import math from collections.abc import Callable -from typing import Any, overload +from typing import Any, cast, overload -from ...backends.adapters import AdapterSchemaMismatchError +from ...backends.adapters import get_io_contract from ...core import ( CBlock, Context, @@ -46,38 +44,22 @@ def requirement_check_to_bool(x: CBlock | ModelOutputThunk | str) -> bool: Raises: json.JSONDecodeError: If `x` is not valid JSON. + ValueError: If the parsed JSON is not an object (e.g. a list, string, or + number). AdapterSchemaMismatchError: If the parsed output does not contain the expected `requirement_check.score` structure, or if the score is not a finite number in the range 0.0-1.0. Callers that previously treated `False` as "requirement not met" must now catch this error separately. """ - output = str(x) - req_dict: dict[str, Any] = json.loads(output) - - # Mirrors the validation in requirement_check() in core.py; Phase 2 will consolidate via IOContract. - req_check = req_dict.get("requirement_check", None) - if not isinstance(req_check, dict): - raise AdapterSchemaMismatchError( - name="requirement-check", - observed_keys=frozenset(req_dict.keys()), - expected_keys=frozenset({"requirement_check"}), - ) - - score = req_check.get("score", None) - if ( - not isinstance(score, (int, float)) - or isinstance(score, bool) # bool subclasses int; exclude it explicitly - or not math.isfinite(score) - or not 0.0 <= score <= 1.0 - ): - raise AdapterSchemaMismatchError( - name="requirement-check", - observed_keys=frozenset(req_check.keys()), - expected_keys=frozenset({"score"}), - ) - - return score > 0.5 + # Delegates to the same `requirement-check` IOContract resolve_adapter() hands + # back to call_intrinsic() — see mellea.backends.adapters.io_contracts. A second, + # independent validation here is exactly the parallel-declaration problem #1516 + # closes; ALoraRequirement (below) is the other production consumer of this + # capability's output alongside core.requirement_check(). + parsed = get_io_contract("requirement-check").parse(str(x)) + score = cast(dict[str, Any], parsed["requirement_check"])["score"] + return cast(float, score) > 0.5 class ALoraRequirement(Requirement, Intrinsic): diff --git a/test/backends/test_adapters/test_core_types.py b/test/backends/test_adapters/test_core_types.py index d3751d891..72fe311ac 100644 --- a/test/backends/test_adapters/test_core_types.py +++ b/test/backends/test_adapters/test_core_types.py @@ -143,6 +143,7 @@ def test_adapter_schema_mismatch_error_format(): assert err.name == "answerability" assert err.observed_keys == observed assert err.expected_keys == expected + assert err.reason is None msg = str(err) assert "answerability" in msg assert "Observed keys:" in msg @@ -153,7 +154,10 @@ def test_adapter_schema_mismatch_error_pickles(): observed = frozenset({"key_a"}) expected = frozenset({"key_b"}) err = AdapterSchemaMismatchError( - name="answerability", observed_keys=observed, expected_keys=expected + name="answerability", + observed_keys=observed, + expected_keys=expected, + reason="schema changed", ) restored = pickle.loads(pickle.dumps(err)) @@ -162,6 +166,8 @@ def test_adapter_schema_mismatch_error_pickles(): assert restored.name == "answerability" assert restored.observed_keys == observed assert restored.expected_keys == expected + assert restored.reason == "schema changed" + assert restored.args == ("answerability", observed, expected) assert str(restored) == str(err) @@ -281,5 +287,5 @@ def test_dict_contract_reports_all_missing_multi_key(): def test_dict_contract_build_prompt_not_implemented(): contract = _DictContract("answerability", frozenset({"answerability"})) - with pytest.raises(NotImplementedError, match="Phase 1"): + with pytest.raises(NotImplementedError, match="build_prompt is not implemented"): contract.build_prompt() diff --git a/test/backends/test_adapters/test_io_contracts.py b/test/backends/test_adapters/test_io_contracts.py new file mode 100644 index 000000000..b02a69441 --- /dev/null +++ b/test/backends/test_adapters/test_io_contracts.py @@ -0,0 +1,231 @@ +# Copyright IBM Corp. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the canonical adapter-function output-contract registry (issue #1516). + +Covers the registry's completeness/exhaustiveness invariant, the fallback contract +for names outside the catalog, and the `_RequirementCheckContract` parsing logic that +replaced `core.requirement_check`'s hand-rolled post-call validation. +""" + +import json +import math +import pathlib + +import pytest + +from mellea.backends.adapters import AdapterSchemaMismatchError, catalog +from mellea.backends.adapters.catalog import ( + IntrinsicsCatalogEntry, + known_intrinsic_names, +) +from mellea.backends.adapters.io_contracts import ( + _BUILTIN_INTRINSIC_NAMES, + _INTRINSIC_IO_CONTRACTS, + get_io_contract, +) + +_INTRINSIC_TESTDATA = ( + pathlib.Path(__file__).resolve().parents[2] + / "stdlib" + / "components" + / "intrinsic" + / "testdata" +) + +# --------------------------------------------------------------------------- +# Registry completeness +# --------------------------------------------------------------------------- + + +def test_registry_covers_every_catalog_name(): + """Every catalogued adapter function must have a declared contract. + + Regression guard for the acceptance criterion that no adapter reachable + through `resolve_adapter` carries an unimplemented placeholder contract. + """ + missing = _BUILTIN_INTRINSIC_NAMES - set(_INTRINSIC_IO_CONTRACTS) + assert missing == set() + + +def test_registry_has_no_orphan_keys(): + """A contract keyed to a name absent from the catalog would be dead code. + + Guards the reverse direction of the completeness invariant: an orphan key + would be served by `get_io_contract` ahead of the permissive fallback if a + later adapter ever registered under that name. + """ + orphans = set(_INTRINSIC_IO_CONTRACTS) - _BUILTIN_INTRINSIC_NAMES + assert orphans == set() + + +def test_get_io_contract_returns_declared_instance_for_known_names(): + for name in _BUILTIN_INTRINSIC_NAMES: + assert get_io_contract(name) is _INTRINSIC_IO_CONTRACTS[name] + + +def test_registry_invariant_ignores_runtime_custom_catalogue_entry(monkeypatch): + """A custom adapter must not make the built-in registry checks order-dependent.""" + custom_name = "custom-user-adapter" + custom_entry = IntrinsicsCatalogEntry( + name=custom_name, repo_id="example/custom-user-adapter", revision="main" + ) + monkeypatch.setattr( + catalog, + "_INTRINSICS_CATALOG", + {**catalog._INTRINSICS_CATALOG, custom_name: custom_entry}, + ) + + assert custom_name in known_intrinsic_names() + assert custom_name not in _BUILTIN_INTRINSIC_NAMES + assert _BUILTIN_INTRINSIC_NAMES - set(_INTRINSIC_IO_CONTRACTS) == set() + + +def test_get_io_contract_falls_back_permissively_for_unknown_names(): + """A name outside the catalog (e.g. a `CustomIntrinsicAdapter`) must not raise.""" + contract = get_io_contract("some-custom-user-adapter") + result = contract.parse(json.dumps({"anything": "goes"})) + assert result == {"anything": "goes"} + + +def test_get_io_contract_fallback_still_rejects_non_dict_output(): + contract = get_io_contract("some-custom-user-adapter") + with pytest.raises(ValueError, match="must be a JSON object"): + contract.parse(json.dumps(["not", "a", "dict"])) + + +# --------------------------------------------------------------------------- +# _RequirementCheckContract — replaces core.requirement_check's hand-rolled validation +# --------------------------------------------------------------------------- + + +@pytest.fixture +def contract(): + return _INTRINSIC_IO_CONTRACTS["requirement-check"] + + +def test_valid_score_returned(contract): + result = contract.parse(json.dumps({"requirement_check": {"score": 0.8}})) + assert result["requirement_check"]["score"] == pytest.approx(0.8) # type: ignore[index] + + +def test_missing_requirement_check_key_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"other_field": 0.9})) + + +def test_requirement_check_not_a_dict_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": None})) + + +def test_requirement_check_list_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": []})) + + +def test_missing_score_key_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"other_key": 0.9}})) + + +def test_null_score_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"score": None}})) + + +def test_string_score_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"score": "0.9"}})) + + +def test_bool_score_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"score": True}})) + + +def test_nan_score_raises(contract): + # json.dumps(nan) emits the non-standard `NaN` token, which json.loads accepts. + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"score": math.nan}})) + + +def test_inf_score_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"score": math.inf}})) + + +def test_score_above_range_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"score": 1.5}})) + + +def test_score_below_range_raises(contract): + with pytest.raises(AdapterSchemaMismatchError): + contract.parse(json.dumps({"requirement_check": {"score": -0.1}})) + + +def test_boundary_score_zero(contract): + result = contract.parse(json.dumps({"requirement_check": {"score": 0.0}})) + assert result["requirement_check"]["score"] == pytest.approx(0.0) # type: ignore[index] + + +def test_boundary_score_one(contract): + result = contract.parse(json.dumps({"requirement_check": {"score": 1.0}})) + assert result["requirement_check"]["score"] == pytest.approx(1.0) # type: ignore[index] + + +def test_requirement_check_rejects_non_dict_output(contract): + with pytest.raises(ValueError, match="must be a JSON object"): + contract.parse(json.dumps(["not", "a", "dict"])) + + +# --------------------------------------------------------------------------- +# context-attribution — recorded model output against the real _ListContract +# +# The GPU-gated equivalents (test_find_context_attributions and +# test_find_context_attributions_resolve in test_core.py) are xfail(strict=False) +# for unrelated non-determinism, so they give no CI signal on schema drift. This +# feeds the exact recorded output through the contract without a GPU. +# --------------------------------------------------------------------------- + + +def test_context_attribution_contract_accepts_recorded_model_output(): + fixture = _INTRINSIC_TESTDATA / "output_json" / "context-attribution.json" + completion = json.loads(fixture.read_text(encoding="utf-8")) + raw = completion["choices"][0]["message"]["content"] + + result = _INTRINSIC_IO_CONTRACTS["context-attribution"].parse(raw) + + assert len(result["items"]) == 7 # type: ignore[arg-type] + assert result["items"][0]["attribution_msg_index"] is None # type: ignore[index] + + +def test_context_attribution_contract_rejects_missing_item_key(): + """A record missing a required item key raises AdapterSchemaMismatchError.""" + fixture = _INTRINSIC_TESTDATA / "output_json" / "context-attribution.json" + completion = json.loads(fixture.read_text(encoding="utf-8")) + raw = completion["choices"][0]["message"]["content"] + items = json.loads(raw) + del items[0]["attribution_text"] + + with pytest.raises(AdapterSchemaMismatchError): + _INTRINSIC_IO_CONTRACTS["context-attribution"].parse(json.dumps(items)) + + +@pytest.mark.parametrize( + ("output", "reason"), + [ + ({"other": "value"}, "neither `label` nor `score` was present"), + ({"label": "Yes", "score": "No"}, "both `label` and `score` were present"), + ], +) +def test_policy_guardrails_contract_distinguishes_exclusivity_failures(output, reason): + """Neither-key and both-key output errors must identify the failed condition.""" + contract = _INTRINSIC_IO_CONTRACTS["policy-guardrails"] + + with pytest.raises(AdapterSchemaMismatchError) as exc_info: + contract.parse(json.dumps(output)) + + assert exc_info.value.reason == reason + assert reason in str(exc_info.value) diff --git a/test/backends/test_adapters/test_shims.py b/test/backends/test_adapters/test_shims.py index 98801a75d..457810dc0 100644 --- a/test/backends/test_adapters/test_shims.py +++ b/test/backends/test_adapters/test_shims.py @@ -15,7 +15,12 @@ import pytest -from mellea.backends.adapters import Adapter, EmbeddedIntrinsicAdapter, IntrinsicAdapter +from mellea.backends.adapters import ( + Adapter, + EmbeddedIntrinsicAdapter, + IntrinsicAdapter, + get_io_contract, +) from mellea.backends.adapters._core import Identity, LocalFileBinding from mellea.backends.adapters.adapter import AdapterMixin from mellea.backends.adapters.catalog import AdapterType, IntrinsicsCatalogEntry @@ -92,6 +97,21 @@ def test_embedded_identity_populated(): assert adapter.identity.adapter_type == "alora" +def test_embedded_carries_the_declared_io_contract_not_a_shim(): + """resolve_adapter()'s shim path must carry a real contract (issue #1516). + + Regression guard: without this, reverting `get_io_contract(intrinsic_name)` + back to the Phase 1 `_ShimIOContract()` placeholder would pass every other + test in this file, since none of them inspect `.io_contract`. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + adapter = EmbeddedIntrinsicAdapter( + "answerability", config={}, technology="alora" + ) + assert adapter.io_contract is get_io_contract("answerability") + + def test_embedded_identity_lora_technology(): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) @@ -170,6 +190,17 @@ def test_intrinsic_identity_populated(): assert adapter.identity.adapter_type == "alora" +def test_intrinsic_carries_the_declared_io_contract_not_a_shim(): + """resolve_adapter()'s shim path must carry a real contract (issue #1516). + + Regression guard: without this, reverting `get_io_contract(intrinsic_name)` + back to the Phase 1 `_ShimIOContract()` placeholder would pass every other + test in this file, since none of them inspect `.io_contract`. + """ + adapter = _make_intrinsic_adapter("answerability") + assert adapter.io_contract is get_io_contract("answerability") + + def test_intrinsic_identity_lora_adapter_type(): with ( patch( diff --git a/test/stdlib/components/intrinsic/test_core_contracts.py b/test/stdlib/components/intrinsic/test_core_contracts.py new file mode 100644 index 000000000..1a1b8fec0 --- /dev/null +++ b/test/stdlib/components/intrinsic/test_core_contracts.py @@ -0,0 +1,109 @@ +# Copyright IBM Corp. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Wiring tests for check_certainty and find_context_attributions (issue #1516). + +Both helpers moved from raw `json.loads` to registry-contract validation when +`call_intrinsic` started parsing via the resolved adapter's contract, and +`find_context_attributions` additionally changed return shape (unwrap of +`"items"`). Their other tests — GPU-gated qualitative/xfail tests in +`test_core.py` plus opt-in e2e runs in `docs/examples` — give no CI signal on +that wiring, so these stub `mfuncs.act` and `backend.resolve_adapter` (the +`test_core_schema.py` pattern) and exercise the helper boundary itself. +""" + +import json +import pathlib +from unittest.mock import MagicMock + +import pytest + +from mellea.backends.adapters import AdapterSchemaMismatchError, get_io_contract +from mellea.stdlib.components import Message +from mellea.stdlib.components.intrinsic import _util, core +from mellea.stdlib.context import ChatContext + + +def _stub_model_call(raw_output: str, monkeypatch, intrinsic_name: str) -> MagicMock: + """Replace `mfuncs.act` to return *raw_output* and stub `resolve_adapter` + to carry *intrinsic_name*'s registry contract. Returns the backend mock.""" + backend = MagicMock() + backend.resolve_adapter.return_value = MagicMock( + io_contract=get_io_contract(intrinsic_name) + ) + + def fake_act(_intrinsic, context, _backend, *, model_options=None, **_kwargs): + thunk = MagicMock() + thunk.is_computed.return_value = True + thunk.value = raw_output + return thunk, context + + monkeypatch.setattr(_util.mfuncs, "act", fake_act) + return backend + + +def test_check_certainty_resolves_and_parses_via_registry_contract(monkeypatch): + """check_certainty must resolve `uncertainty` and parse its contract output.""" + backend = _stub_model_call( + json.dumps({"certainty": 0.9}), monkeypatch, "uncertainty" + ) + context = ChatContext().add(Message("user", "hi")) + + assert core.check_certainty(context, backend) == pytest.approx(0.9) + backend.resolve_adapter.assert_called_once_with("uncertainty") + + +def test_check_certainty_missing_certainty_key_raises(monkeypatch): + """A missing `certainty` key must raise AdapterSchemaMismatchError.""" + backend = _stub_model_call( + json.dumps({"wrong_key": 0.9}), monkeypatch, "uncertainty" + ) + context = ChatContext().add(Message("user", "hi")) + + with pytest.raises(AdapterSchemaMismatchError): + core.check_certainty(context, backend) + + +def test_check_certainty_rejects_non_object_output(monkeypatch): + """A top-level JSON array is a ValueError, not a schema mismatch.""" + backend = _stub_model_call(json.dumps([0.9]), monkeypatch, "uncertainty") + context = ChatContext().add(Message("user", "hi")) + + with pytest.raises(ValueError, match="must be a JSON object"): + core.check_certainty(context, backend) + + +def test_find_context_attributions_returns_items_from_recorded_output(monkeypatch): + """The `result_json["items"]` unwrap must survive the helper boundary.""" + fixture = ( + pathlib.Path(__file__).resolve().parent + / "testdata" + / "output_json" + / "context-attribution.json" + ) + completion = json.loads(fixture.read_text(encoding="utf-8")) + raw = completion["choices"][0]["message"]["content"] + + backend = _stub_model_call(raw, monkeypatch, "context-attribution") + context = ChatContext().add(Message("user", "hi")) + + result = core.find_context_attributions( + "The answer is 42.", ["A document."], context, backend + ) + + assert len(result) == 7 + assert result[0]["attribution_msg_index"] is None + backend.resolve_adapter.assert_called_once_with("context-attribution") + + +def test_find_context_attributions_rejects_non_array_output(monkeypatch): + """A top-level JSON object is a ValueError for the list contract.""" + backend = _stub_model_call( + json.dumps({"not": "a list"}), monkeypatch, "context-attribution" + ) + context = ChatContext().add(Message("user", "hi")) + + with pytest.raises(ValueError, match="must be a JSON array"): + core.find_context_attributions( + "The answer is 42.", ["A document."], context, backend + ) diff --git a/test/stdlib/components/intrinsic/test_core_schema.py b/test/stdlib/components/intrinsic/test_core_schema.py index fa6607be4..8cb0fcfd5 100644 --- a/test/stdlib/components/intrinsic/test_core_schema.py +++ b/test/stdlib/components/intrinsic/test_core_schema.py @@ -1,93 +1,123 @@ # Copyright IBM Corp. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for requirement_check schema validation in core.py. - -These tests mock call_intrinsic so they run without a GPU or HF backend. +"""Integration-style unit tests for `core.requirement_check`'s schema validation. + +Issue #1516 replaced the hand-rolled score-range validation `requirement_check` used +to run after `call_intrinsic` with the `requirement-check` capability's declared +`IOContract`, obtained from the adapter that `resolve_adapter()` returns rather than passed +as a parallel argument. These tests exercise that real path — `call_intrinsic` and +`IOContract.parse` both run against a stubbed `resolve_adapter` return value — by +mocking `mfuncs.act` (the actual model call) and `backend.resolve_adapter` (a bare +`MagicMock`), not `call_intrinsic` itself. `resolve_adapter`'s own lazy-registration +and catalogue-lookup logic is exercised separately in `test/backends/test_adapters/`. + +Score-range edge cases (NaN, bool, out-of-range, ...) are covered directly against +the contract in `test/backends/test_adapters/test_io_contracts.py`; this file checks +that `core.requirement_check` is wired to that contract end-to-end. """ -import math -from unittest.mock import patch +import json +from unittest.mock import MagicMock import pytest -from mellea.backends.adapters import AdapterSchemaMismatchError -from mellea.stdlib.components.intrinsic import core +from mellea.backends.adapters import ( + Adapter, + AdapterSchemaMismatchError, + Identity, + LocalFileBinding, + get_io_contract, +) +from mellea.stdlib.components import Message +from mellea.stdlib.components.intrinsic import _util, core from mellea.stdlib.context import ChatContext -_CTX = ChatContext() -_BACKEND = object() _REQUIREMENT = "must be polite" -_PATCH = "mellea.stdlib.components.intrinsic.core.call_intrinsic" - - -def _call(result_json: dict) -> float: - with patch(_PATCH, return_value=result_json): - return core.requirement_check(_CTX, _BACKEND, _REQUIREMENT) # type: ignore[arg-type] +_REQUIREMENT_CHECK_ADAPTER = Adapter( + identity=Identity("requirement-check", "alora", capability="requirement_check"), + io_contract=get_io_contract("requirement-check"), + weights=LocalFileBinding(), +) -def test_valid_score_returned(): - assert _call({"requirement_check": {"score": 0.8}}) == pytest.approx(0.8) - - -def test_missing_requirement_check_key_raises(): - with pytest.raises(AdapterSchemaMismatchError): - _call({"other_field": 0.9}) +def _call(result_dict: dict, monkeypatch) -> float: + backend = MagicMock() + backend.resolve_adapter.return_value = _REQUIREMENT_CHECK_ADAPTER + def fake_act(_intrinsic, context, _backend, *, model_options=None, **_kwargs): + thunk = MagicMock() + thunk.is_computed.return_value = True + thunk.value = json.dumps(result_dict) + return thunk, context -def test_null_requirement_check_raises(): - with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": None}) + monkeypatch.setattr(_util.mfuncs, "act", fake_act) + context = ChatContext().add(Message("user", "hi")) + return core.requirement_check(context, backend, _REQUIREMENT) -def test_list_requirement_check_raises(): - with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": []}) +def test_valid_score_returned(monkeypatch): + assert _call({"requirement_check": {"score": 0.8}}, monkeypatch) == pytest.approx( + 0.8 + ) -def test_missing_score_key_raises(): - with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"other_key": 0.9}}) +def test_boundary_score_zero(monkeypatch): + assert _call({"requirement_check": {"score": 0.0}}, monkeypatch) == pytest.approx( + 0.0 + ) -def test_null_score_raises(): - with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"score": None}}) +def test_boundary_score_one(monkeypatch): + assert _call({"requirement_check": {"score": 1.0}}, monkeypatch) == pytest.approx( + 1.0 + ) -def test_string_score_raises(): +def test_missing_requirement_check_key_raises(monkeypatch): with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"score": "0.9"}}) + _call({"other_field": 0.9}, monkeypatch) -def test_bool_score_raises(): +def test_missing_score_key_raises(monkeypatch): with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"score": True}}) + _call({"requirement_check": {"other_key": 0.9}}, monkeypatch) -def test_nan_score_raises(): +def test_score_above_range_raises(monkeypatch): with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"score": math.nan}}) + _call({"requirement_check": {"score": 1.5}}, monkeypatch) -def test_inf_score_raises(): +def test_score_below_range_raises(monkeypatch): with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"score": math.inf}}) + _call({"requirement_check": {"score": -0.1}}, monkeypatch) -def test_score_above_range_raises(): - with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"score": 1.5}}) +def test_requirement_check_resolves_adapter_by_name(monkeypatch): + """core.requirement_check must resolve the `requirement-check` capability.""" + backend = MagicMock() + backend.resolve_adapter.return_value = _REQUIREMENT_CHECK_ADAPTER + def fake_act(_intrinsic, context, _backend, *, model_options=None, **_kwargs): + thunk = MagicMock() + thunk.is_computed.return_value = True + thunk.value = json.dumps({"requirement_check": {"score": 0.5}}) + return thunk, context -def test_score_below_range_raises(): - with pytest.raises(AdapterSchemaMismatchError): - _call({"requirement_check": {"score": -0.1}}) + monkeypatch.setattr(_util.mfuncs, "act", fake_act) + context = ChatContext().add(Message("user", "hi")) + core.requirement_check(context, backend, _REQUIREMENT) + backend.resolve_adapter.assert_called_once_with("requirement-check") -def test_boundary_score_zero(): - assert _call({"requirement_check": {"score": 0.0}}) == pytest.approx(0.0) +def test_requirement_check_adapter_carries_registry_contract(): + """The stub must carry the registry's declared contract, not a local copy.""" + assert _REQUIREMENT_CHECK_ADAPTER.io_contract is get_io_contract( + "requirement-check" + ) -def test_boundary_score_one(): - assert _call({"requirement_check": {"score": 1.0}}) == pytest.approx(1.0) +def test_core_module_does_not_define_requirement_check_adapter(): + """The adapter stub belongs to this test, not production core code.""" + assert not hasattr(core, "_REQUIREMENT_CHECK_ADAPTER") diff --git a/test/stdlib/components/intrinsic/test_guardian_deprecation.py b/test/stdlib/components/intrinsic/test_guardian_deprecation.py index 59a2cadef..1984343d7 100644 --- a/test/stdlib/components/intrinsic/test_guardian_deprecation.py +++ b/test/stdlib/components/intrinsic/test_guardian_deprecation.py @@ -21,9 +21,7 @@ def capture_kwargs(monkeypatch): """Replace call_intrinsic with a spy that returns a stub yes=1.0 result.""" captured: dict = {} - def fake_call_intrinsic( - name, context, backend, /, kwargs=None, model_options=None, io_contract=None - ): + def fake_call_intrinsic(name, context, backend, /, kwargs=None, model_options=None): captured["name"] = name captured["kwargs"] = kwargs return {"guardian": {"score": 1.0}} diff --git a/test/stdlib/components/intrinsic/test_guardian_documents.py b/test/stdlib/components/intrinsic/test_guardian_documents.py index 9201082a6..5e12463cc 100644 --- a/test/stdlib/components/intrinsic/test_guardian_documents.py +++ b/test/stdlib/components/intrinsic/test_guardian_documents.py @@ -20,9 +20,7 @@ def capture_intrinsic(monkeypatch): """Spy that replaces call_intrinsic and captures what it receives.""" captured: dict = {} - def fake_call_intrinsic( - name, context, backend, /, kwargs=None, model_options=None, io_contract=None - ): + def fake_call_intrinsic(name, context, backend, /, kwargs=None, model_options=None): captured["name"] = name captured["context"] = context return {"score": "yes", "correction": "corrected"} @@ -215,7 +213,7 @@ def _make_capture(monkeypatch, result: dict): """Patch call_intrinsic to capture (name, model_options) and return result.""" calls: list[tuple] = [] - def _fake(name, ctx, backend, /, kwargs=None, model_options=None, io_contract=None): + def _fake(name, ctx, backend, /, kwargs=None, model_options=None): calls.append((name, model_options)) return result diff --git a/test/stdlib/components/intrinsic/test_guardian_io_contract.py b/test/stdlib/components/intrinsic/test_guardian_io_contract.py index e230539ba..5b194c02f 100644 --- a/test/stdlib/components/intrinsic/test_guardian_io_contract.py +++ b/test/stdlib/components/intrinsic/test_guardian_io_contract.py @@ -1,10 +1,13 @@ # Copyright IBM Corp. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for IOContract validation in guardian.py (Epic #929 Phase 1). +"""Unit tests for the guardian adapter functions' output contracts (Epic #929, +issue #1516). -Tests the `parse()` method of each IOContract subclass directly — no backend, -no GPU, no model download required. Two tests per helper: +The contracts are declared in `mellea.backends.adapters.io_contracts` and +looked up by catalog name via `get_io_contract`. Tests the `parse()` method +of each contract directly — no backend, no GPU, no model download required. +Two tests per helper: - `test__contract_enforced` — output missing a required field raises :class:`~mellea.backends.adapters.AdapterSchemaMismatchError`. @@ -17,14 +20,12 @@ import pytest -from mellea.backends.adapters import AdapterMixin, AdapterSchemaMismatchError -from mellea.stdlib.components.intrinsic import guardian -from mellea.stdlib.components.intrinsic.guardian import ( - _FACTUALITY_CORRECTION_ADAPTER, - _FACTUALITY_DETECTION_ADAPTER, - _GUARDIAN_CHECK_ADAPTER, - _POLICY_GUARDRAILS_ADAPTER, +from mellea.backends.adapters import ( + AdapterMixin, + AdapterSchemaMismatchError, + get_io_contract, ) +from mellea.stdlib.components.intrinsic import guardian from mellea.stdlib.context import ChatContext # --------------------------------------------------------------------------- @@ -33,8 +34,9 @@ def test_policy_guardrails_contract_enforced_neither_key() -> None: + contract = get_io_contract("policy-guardrails") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _POLICY_GUARDRAILS_ADAPTER.io_contract.parse(json.dumps({"wrong_key": "value"})) + contract.parse(json.dumps({"wrong_key": "value"})) err = exc_info.value assert err.name == "policy-guardrails" assert "label" in err.expected_keys @@ -42,31 +44,29 @@ def test_policy_guardrails_contract_enforced_neither_key() -> None: def test_policy_guardrails_contract_enforced_both_keys() -> None: + contract = get_io_contract("policy-guardrails") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _POLICY_GUARDRAILS_ADAPTER.io_contract.parse( - json.dumps({"label": "Yes", "score": "Yes"}) - ) + contract.parse(json.dumps({"label": "Yes", "score": "Yes"})) err = exc_info.value assert err.name == "policy-guardrails" def test_policy_guardrails_forward_compat_label() -> None: - result = _POLICY_GUARDRAILS_ADAPTER.io_contract.parse( - json.dumps({"label": "Yes", "extra": "ignored"}) - ) + contract = get_io_contract("policy-guardrails") + result = contract.parse(json.dumps({"label": "Yes", "extra": "ignored"})) assert result["label"] == "Yes" def test_policy_guardrails_forward_compat_score() -> None: - result = _POLICY_GUARDRAILS_ADAPTER.io_contract.parse( - json.dumps({"score": "No", "extra": "ignored"}) - ) + contract = get_io_contract("policy-guardrails") + result = contract.parse(json.dumps({"score": "No", "extra": "ignored"})) assert result["score"] == "No" def test_policy_guardrails_rejects_non_dict() -> None: + contract = get_io_contract("policy-guardrails") with pytest.raises(ValueError, match="must be a JSON object"): - _POLICY_GUARDRAILS_ADAPTER.io_contract.parse(json.dumps(["not", "a", "dict"])) + contract.parse(json.dumps(["not", "a", "dict"])) # --------------------------------------------------------------------------- @@ -75,26 +75,27 @@ def test_policy_guardrails_rejects_non_dict() -> None: def test_guardian_check_contract_enforced_missing_guardian_key() -> None: + contract = get_io_contract("guardian-core") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _GUARDIAN_CHECK_ADAPTER.io_contract.parse(json.dumps({"wrong_key": 0.5})) + contract.parse(json.dumps({"wrong_key": 0.5})) err = exc_info.value assert err.name == "guardian-core" assert "guardian" in err.expected_keys def test_guardian_check_contract_enforced_missing_score_in_guardian() -> None: + contract = get_io_contract("guardian-core") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _GUARDIAN_CHECK_ADAPTER.io_contract.parse( - json.dumps({"guardian": {"wrong_key": 0.5}}) - ) + contract.parse(json.dumps({"guardian": {"wrong_key": 0.5}})) err = exc_info.value assert err.name == "guardian-core" assert "score" in err.expected_keys def test_guardian_check_contract_enforced_guardian_not_dict() -> None: + contract = get_io_contract("guardian-core") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _GUARDIAN_CHECK_ADAPTER.io_contract.parse(json.dumps({"guardian": 0.8})) + contract.parse(json.dumps({"guardian": 0.8})) err = exc_info.value assert err.name == "guardian-core" assert "score" in err.expected_keys @@ -102,7 +103,8 @@ def test_guardian_check_contract_enforced_guardian_not_dict() -> None: def test_guardian_check_forward_compat() -> None: - result = _GUARDIAN_CHECK_ADAPTER.io_contract.parse( + contract = get_io_contract("guardian-core") + result = contract.parse( json.dumps({"guardian": {"score": 0.9}, "extra": "ignored"}) ) assert isinstance(result["guardian"], dict) @@ -110,8 +112,9 @@ def test_guardian_check_forward_compat() -> None: def test_guardian_check_rejects_non_dict() -> None: + contract = get_io_contract("guardian-core") with pytest.raises(ValueError, match="must be a JSON object"): - _GUARDIAN_CHECK_ADAPTER.io_contract.parse(json.dumps([0.5])) + contract.parse(json.dumps([0.5])) # --------------------------------------------------------------------------- @@ -120,25 +123,24 @@ def test_guardian_check_rejects_non_dict() -> None: def test_factuality_detection_contract_enforced() -> None: + contract = get_io_contract("factuality-detection") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _FACTUALITY_DETECTION_ADAPTER.io_contract.parse( - json.dumps({"wrong_key": "yes"}) - ) + contract.parse(json.dumps({"wrong_key": "yes"})) err = exc_info.value assert err.name == "factuality-detection" assert "score" in err.expected_keys def test_factuality_detection_forward_compat() -> None: - result = _FACTUALITY_DETECTION_ADAPTER.io_contract.parse( - json.dumps({"score": "yes", "confidence": 0.9}) - ) + contract = get_io_contract("factuality-detection") + result = contract.parse(json.dumps({"score": "yes", "confidence": 0.9})) assert result["score"] == "yes" def test_factuality_detection_rejects_non_dict() -> None: + contract = get_io_contract("factuality-detection") with pytest.raises(ValueError, match="must be a JSON object"): - _FACTUALITY_DETECTION_ADAPTER.io_contract.parse(json.dumps("yes")) + contract.parse(json.dumps("yes")) # --------------------------------------------------------------------------- @@ -147,27 +149,26 @@ def test_factuality_detection_rejects_non_dict() -> None: def test_factuality_correction_contract_enforced() -> None: + contract = get_io_contract("factuality-correction") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _FACTUALITY_CORRECTION_ADAPTER.io_contract.parse( - json.dumps({"wrong_key": "corrected text"}) - ) + contract.parse(json.dumps({"wrong_key": "corrected text"})) err = exc_info.value assert err.name == "factuality-correction" assert "correction" in err.expected_keys def test_factuality_correction_forward_compat() -> None: - result = _FACTUALITY_CORRECTION_ADAPTER.io_contract.parse( + contract = get_io_contract("factuality-correction") + result = contract.parse( json.dumps({"correction": "The correct answer is 42.", "score": 0.95}) ) assert result["correction"] == "The correct answer is 42." def test_factuality_correction_rejects_non_dict() -> None: + contract = get_io_contract("factuality-correction") with pytest.raises(ValueError, match="must be a JSON object"): - _FACTUALITY_CORRECTION_ADAPTER.io_contract.parse( - json.dumps(["not", "a", "dict"]) - ) + contract.parse(json.dumps(["not", "a", "dict"])) # --------------------------------------------------------------------------- @@ -176,26 +177,30 @@ def test_factuality_correction_rejects_non_dict() -> None: def test_policy_guardrails_error_mentions_adapter_name() -> None: + contract = get_io_contract("policy-guardrails") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _POLICY_GUARDRAILS_ADAPTER.io_contract.parse(json.dumps({})) + contract.parse(json.dumps({})) assert exc_info.value.name == "policy-guardrails" def test_guardian_check_error_mentions_adapter_name() -> None: + contract = get_io_contract("guardian-core") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _GUARDIAN_CHECK_ADAPTER.io_contract.parse(json.dumps({})) + contract.parse(json.dumps({})) assert exc_info.value.name == "guardian-core" def test_factuality_detection_error_mentions_adapter_name() -> None: + contract = get_io_contract("factuality-detection") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _FACTUALITY_DETECTION_ADAPTER.io_contract.parse(json.dumps({})) + contract.parse(json.dumps({})) assert exc_info.value.name == "factuality-detection" def test_factuality_correction_error_mentions_adapter_name() -> None: + contract = get_io_contract("factuality-correction") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _FACTUALITY_CORRECTION_ADAPTER.io_contract.parse(json.dumps({})) + contract.parse(json.dumps({})) assert exc_info.value.name == "factuality-correction" @@ -207,9 +212,7 @@ def test_factuality_correction_error_mentions_adapter_name() -> None: def test_policy_guardrails_score_branch(monkeypatch) -> None: """policy_guardrails returns the `score` value when the adapter omits `label`.""" - def fake_call_intrinsic( - name, context, backend, /, kwargs=None, model_options=None, io_contract=None - ): + def fake_call_intrinsic(name, context, backend, /, kwargs=None, model_options=None): return {"score": "No"} monkeypatch.setattr(guardian, "call_intrinsic", fake_call_intrinsic) diff --git a/test/stdlib/components/intrinsic/test_rag_contracts.py b/test/stdlib/components/intrinsic/test_rag_contracts.py index aaa2af32f..79026b112 100644 --- a/test/stdlib/components/intrinsic/test_rag_contracts.py +++ b/test/stdlib/components/intrinsic/test_rag_contracts.py @@ -1,10 +1,12 @@ # Copyright IBM Corp. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for IOContract validation in rag.py (Epic #929 Phase 1). +"""Unit tests for the rag adapter functions' output contracts (Epic #929, issue #1516). -Tests the `parse()` method of each IOContract subclass directly — no backend, -no GPU, no model download required. Two tests per helper: +The contracts are declared in `mellea.backends.adapters.io_contracts` and +looked up by catalog name via `get_io_contract`. Tests the `parse()` method +of each contract directly — no backend, no GPU, no model download required. +Two tests per helper: - `test__contract_enforced` — output missing a required field raises :class:`~mellea.backends.adapters.AdapterSchemaMismatchError`. @@ -16,15 +18,7 @@ import pytest -from mellea.backends.adapters import AdapterSchemaMismatchError -from mellea.stdlib.components.intrinsic.rag import ( - _ANSWERABILITY_ADAPTER, - _CITATIONS_ADAPTER, - _CONTEXT_RELEVANCE_ADAPTER, - _HALLUCINATION_ADAPTER, - _QUERY_CLARIFY_ADAPTER, - _QUERY_REWRITE_ADAPTER, -) +from mellea.backends.adapters import AdapterSchemaMismatchError, get_io_contract # --------------------------------------------------------------------------- # check_answerability @@ -32,15 +26,17 @@ def test_check_answerability_contract_enforced() -> None: + contract = get_io_contract("answerability") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _ANSWERABILITY_ADAPTER.io_contract.parse(json.dumps({"wrong_key": "value"})) + contract.parse(json.dumps({"wrong_key": "value"})) err = exc_info.value assert err.name == "answerability" assert "answerability" in err.expected_keys def test_check_answerability_forward_compat() -> None: - result = _ANSWERABILITY_ADAPTER.io_contract.parse( + contract = get_io_contract("answerability") + result = contract.parse( json.dumps({"answerability": "answerable", "extra": "ignored"}) ) assert result["answerability"] == "answerable" @@ -52,15 +48,17 @@ def test_check_answerability_forward_compat() -> None: def test_rewrite_question_contract_enforced() -> None: + contract = get_io_contract("query_rewrite") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _QUERY_REWRITE_ADAPTER.io_contract.parse(json.dumps({"wrong_key": "value"})) + contract.parse(json.dumps({"wrong_key": "value"})) err = exc_info.value assert err.name == "query_rewrite" assert "rewritten_question" in err.expected_keys def test_rewrite_question_forward_compat() -> None: - result = _QUERY_REWRITE_ADAPTER.io_contract.parse( + contract = get_io_contract("query_rewrite") + result = contract.parse( json.dumps({"rewritten_question": "new query?", "confidence": 0.9}) ) assert result["rewritten_question"] == "new query?" @@ -72,17 +70,17 @@ def test_rewrite_question_forward_compat() -> None: def test_clarify_query_contract_enforced() -> None: + contract = get_io_contract("query_clarification") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _QUERY_CLARIFY_ADAPTER.io_contract.parse(json.dumps({"wrong_key": "value"})) + contract.parse(json.dumps({"wrong_key": "value"})) err = exc_info.value assert err.name == "query_clarification" assert "clarification" in err.expected_keys def test_clarify_query_forward_compat() -> None: - result = _QUERY_CLARIFY_ADAPTER.io_contract.parse( - json.dumps({"clarification": "CLEAR", "score": 1.0}) - ) + contract = get_io_contract("query_clarification") + result = contract.parse(json.dumps({"clarification": "CLEAR", "score": 1.0})) assert result["clarification"] == "CLEAR" @@ -104,8 +102,9 @@ def test_clarify_query_forward_compat() -> None: def test_find_citations_contract_enforced() -> None: bad_item = {k: v for k, v in _GOOD_CITATION.items() if k != "citation_doc_id"} + contract = get_io_contract("citations") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _CITATIONS_ADAPTER.io_contract.parse(json.dumps([bad_item])) + contract.parse(json.dumps([bad_item])) err = exc_info.value assert err.name == "citations" assert "citation_doc_id" in err.expected_keys @@ -113,7 +112,8 @@ def test_find_citations_contract_enforced() -> None: def test_find_citations_forward_compat() -> None: extra_item = {**_GOOD_CITATION, "extra_field": "ignored"} - result = _CITATIONS_ADAPTER.io_contract.parse(json.dumps([extra_item])) + contract = get_io_contract("citations") + result = contract.parse(json.dumps([extra_item])) assert result["items"][0]["citation_doc_id"] == "0" # type: ignore[index] @@ -123,17 +123,17 @@ def test_find_citations_forward_compat() -> None: def test_check_context_relevance_contract_enforced() -> None: + contract = get_io_contract("context_relevance") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _CONTEXT_RELEVANCE_ADAPTER.io_contract.parse(json.dumps({"wrong_key": "value"})) + contract.parse(json.dumps({"wrong_key": "value"})) err = exc_info.value assert err.name == "context_relevance" assert "context_relevance" in err.expected_keys def test_check_context_relevance_forward_compat() -> None: - result = _CONTEXT_RELEVANCE_ADAPTER.io_contract.parse( - json.dumps({"context_relevance": "relevant", "score": 0.8}) - ) + contract = get_io_contract("context_relevance") + result = contract.parse(json.dumps({"context_relevance": "relevant", "score": 0.8})) assert result["context_relevance"] == "relevant" @@ -153,8 +153,9 @@ def test_check_context_relevance_forward_compat() -> None: def test_flag_hallucinated_content_contract_enforced() -> None: bad_item = {k: v for k, v in _GOOD_SPAN.items() if k != "explanation"} + contract = get_io_contract("hallucination_detection") with pytest.raises(AdapterSchemaMismatchError) as exc_info: - _HALLUCINATION_ADAPTER.io_contract.parse(json.dumps([bad_item])) + contract.parse(json.dumps([bad_item])) err = exc_info.value assert err.name == "hallucination_detection" assert "explanation" in err.expected_keys @@ -162,7 +163,8 @@ def test_flag_hallucinated_content_contract_enforced() -> None: def test_flag_hallucinated_content_forward_compat() -> None: extra_item = {**_GOOD_SPAN, "extra_field": "ignored"} - result = _HALLUCINATION_ADAPTER.io_contract.parse(json.dumps([extra_item])) + contract = get_io_contract("hallucination_detection") + result = contract.parse(json.dumps([extra_item])) assert result["items"][0]["faithfulness"] == "faithful" # type: ignore[index] @@ -172,12 +174,14 @@ def test_flag_hallucinated_content_forward_compat() -> None: def test_find_citations_empty_list() -> None: - result = _CITATIONS_ADAPTER.io_contract.parse(json.dumps([])) + contract = get_io_contract("citations") + result = contract.parse(json.dumps([])) assert result == {"items": []} def test_flag_hallucinated_content_empty_list() -> None: - result = _HALLUCINATION_ADAPTER.io_contract.parse(json.dumps([])) + contract = get_io_contract("hallucination_detection") + result = contract.parse(json.dumps([])) assert result == {"items": []} @@ -187,27 +191,30 @@ def test_flag_hallucinated_content_empty_list() -> None: def test_dict_contract_rejects_non_dict() -> None: + contract = get_io_contract("answerability") with pytest.raises(ValueError, match="must be a JSON object"): - _ANSWERABILITY_ADAPTER.io_contract.parse(json.dumps(["not", "a", "dict"])) + contract.parse(json.dumps(["not", "a", "dict"])) def test_dict_contract_error_mentions_adapter_name() -> None: + contract = get_io_contract("answerability") with pytest.raises(ValueError, match="answerability"): - _ANSWERABILITY_ADAPTER.io_contract.parse(json.dumps(42)) + contract.parse(json.dumps(42)) def test_list_contract_rejects_non_list() -> None: + contract = get_io_contract("citations") with pytest.raises(ValueError, match="must be a JSON array"): - _CITATIONS_ADAPTER.io_contract.parse(json.dumps({"not": "a list"})) + contract.parse(json.dumps({"not": "a list"})) def test_list_contract_rejects_non_dict_element() -> None: + contract = get_io_contract("citations") with pytest.raises(ValueError, match="must contain only JSON objects"): - _CITATIONS_ADAPTER.io_contract.parse(json.dumps(["string_element"])) + contract.parse(json.dumps(["string_element"])) def test_list_contract_rejects_non_dict_element_after_valid_item() -> None: + contract = get_io_contract("citations") with pytest.raises(ValueError, match="must contain only JSON objects"): - _CITATIONS_ADAPTER.io_contract.parse( - json.dumps([_GOOD_CITATION, "string_element"]) - ) + contract.parse(json.dumps([_GOOD_CITATION, "string_element"])) diff --git a/test/stdlib/components/intrinsic/test_util_unit.py b/test/stdlib/components/intrinsic/test_util_unit.py index 666c01bb3..1bdadd186 100644 --- a/test/stdlib/components/intrinsic/test_util_unit.py +++ b/test/stdlib/components/intrinsic/test_util_unit.py @@ -1,11 +1,14 @@ # Copyright IBM Corp. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for `call_intrinsic`'s model_options resolution. +"""Unit tests for `call_intrinsic`'s model_options resolution and contract +wiring. Exercises the model_options precedence without a real backend or model — guards against the PR #972 bug class (caller-supplied model_options silently -discarded behind a hardcoded default) resurfacing. +discarded behind a hardcoded default) resurfacing. Also covers the issue #1516 +change: the output contract is taken from the adapter that `resolve_adapter()` +returns, and `call_intrinsic` no longer accepts an `io_contract` kwarg. """ import json @@ -107,3 +110,40 @@ def test_call_intrinsic_rejects_empty_chat_context(monkeypatch): backend.resolve_adapter.assert_not_called() assert calls == [] + + +def test_call_intrinsic_parses_via_resolved_adapters_io_contract(monkeypatch): + """The output contract must come from resolve_adapter's return value, not a + parallel argument (issue #1516).""" + calls: list[dict | None] = [] + monkeypatch.setattr(_util.mfuncs, "act", _fake_act_capturing(calls)) + + resolved_adapter = MagicMock() + resolved_adapter.io_contract.parse.return_value = {"parsed": "by-resolved-adapter"} + backend = MagicMock() + backend.resolve_adapter.return_value = resolved_adapter + context = ChatContext().add(Message("user", "hi")) + + result = _util.call_intrinsic("answerability", context, backend) + + resolved_adapter.io_contract.parse.assert_called_once_with( + json.dumps({"result": "ok"}) + ) + assert result == {"parsed": "by-resolved-adapter"} + + +def test_call_intrinsic_no_longer_accepts_io_contract_kwarg(monkeypatch): + """A caller can no longer pass a separate, possibly-mismatched io_contract.""" + calls: list[dict | None] = [] + monkeypatch.setattr(_util.mfuncs, "act", _fake_act_capturing(calls)) + + backend = MagicMock() + context = ChatContext().add(Message("user", "hi")) + + with pytest.raises(TypeError, match="io_contract"): + _util.call_intrinsic( + "answerability", + context, + backend, + io_contract=MagicMock(), # type: ignore[call-arg] + ) diff --git a/test/stdlib/requirements/test_requirement.py b/test/stdlib/requirements/test_requirement.py index 8f5e79e85..f8d411689 100644 --- a/test/stdlib/requirements/test_requirement.py +++ b/test/stdlib/requirements/test_requirement.py @@ -127,6 +127,16 @@ def test_requirement_check_to_bool_invalid_json(): requirement_check_to_bool("not json") +def test_requirement_check_to_bool_non_object_raises(): + """A top-level JSON array or scalar is a ValueError, not a schema mismatch. + + The replaced code raised an undocumented AttributeError here instead + (`list.get` on the parsed result). + """ + with pytest.raises(ValueError, match="must be a JSON object"): + requirement_check_to_bool("[1, 2]") + + def test_requirement_check_to_bool_nan_score_raises(): """NaN would silently evaluate as False without the finiteness guard.""" with pytest.raises(AdapterSchemaMismatchError):