diff --git a/nemoguardrails/server/experimental/contracts/README.md b/nemoguardrails/server/experimental/contracts/README.md new file mode 100644 index 0000000000..f8ac3798c2 --- /dev/null +++ b/nemoguardrails/server/experimental/contracts/README.md @@ -0,0 +1,107 @@ +# Guard contracts + +A guard contract describes a provider operation's guardrail boundary: which +content is inspected, which values are restricted, and which data remains under +provider authority. It makes that boundary reviewable without reading Python. + +The Python projection models, bindings, and endpoint implementation are the +source of truth. A contract is a description of their policy, not server +configuration or a second policy to maintain by hand. Deployment configuration +selects the rails that inspect supported content. + +This directory defines the experimental document format and an illustrative +example. Provider exports belong with their integrations; the format alone does +not implement an endpoint, an exporter, or runtime YAML loading. + +## Document structure + +Start with [minimal.guard.example.yaml](minimal.guard.example.yaml). It marks +request `prompt` and response `text` as guarded, leaves `model` opaque, and +constrains response `status` to `complete`. It is a format example, not the +output of a provider integration. + +| Section | Describes | +| --- | --- | +| `version`, `operationId`, `profile` | Format revision, provider operation, and framework capability. | +| `request`, `response` | Request and buffered-response payload projections. | +| `stream` | Optional event payloads, classifications, and transport framing. | +| `excluded_stream_events` | Explicitly unsupported event schemas and the reasons for exclusion. | +| `integration.endpoint` | Endpoint metadata such as route, rejection codes, and implementation bindings. | + +Payload projections use a subset of JSON Schema vocabulary: `type`, +`properties`, `required`, `items`, `oneOf`, constants, enums, and bounds. +Guard annotations use the `x-nemo-guardrails` key. Other `x-` extensions are +allowed, but every other key starting with `x-nemo` is reserved and rejected. + +## Field policy + +| Declaration | Meaning | +| --- | --- | +| `classification: guarded` | A text subject, or a container leading to it. | +| `subject` | Identifies text, its user/assistant role, and replacement eligibility. | +| `classification: constrained` | Restricts a value or shape without making it a rail subject. | +| `gate: disabled` | Marks a constrained, null-only field for an unsupported feature. | +| `classification: opaque` | Leaves the value under provider authority, without guardrail inspection. | +| `opaque_fields` | Lists reviewed opaque names on this object without declaring each property. | +| `reason` | Explains a restriction: capability, provider integrity, or projection policy. | + +Opaque does not mean safe or trusted. Its nested content is not independently +reviewed. Opaque names should not overlap declared properties or use wildcards. +`extension: true` identifies a local compatibility field outside the pinned +provider schema, not an exemption from review. + +An object's `required` list controls presence; nullability is separate. Disabled +fields can be omitted when optional, or supplied as null, but cannot enable the +feature. A `default` describes an internal default, not permission to insert +fields into the forwarded provider payload. + +`replaceable` records text replacement eligibility. `replacement_blocked_by` +identifies metadata that can prevent replacement, with a `replacement_reason`. +The integration determines the blocking predicate and whether replacement +outcomes can be executed. Neither marking a subject nor marking it replaceable +enables a detector or a runtime capability. + +## Object and stream metadata + +`model` identifies the Python model; `source` identifies a provider schema +component. `unknown_fields: configurable` marks an object whose unknown-field +handling depends on runtime validation context. Generic JSON Schema validators +do not enforce this annotation. `additionalProperties` remains the ordinary +schema-level object closure rule. Provider-required opaque fields may be left +to provider validation; a projection is not a full provider request validator. + +Object `variant` metadata describes discriminator matching and selected-variant +cardinality. This is distinct from ordinary array length constraints. +`stream_selector_field` identifies a request flag, not streaming support. + +A stream's `oneOf` branches describe event families: `guarded_delta`, `snapshot`, +`opaque`, or `provider_error`. Their variants identify source schemas, matching +rules, and classifier shape names. Missing-text policy is distinct from schema +nullability. Transport metadata describes SSE discriminators and sentinels. +Event exclusions and external-source reasons record scope and provenance; they +do not establish that every provider event has been reviewed. + +Stateful event ordering, lifecycle checks, conditional replacement, and native +error encoding remain implementation behavior. Endpoint symbol references are +descriptive metadata, not instructions to load code from an untrusted document. + +## Validation and stability + +[guard-contract.schema.json](guard-contract.schema.json) validates the document +format and rejects unsupported keys in the reserved guardrail vocabulary. Other +`x-*` schema annotations are allowed. The schema can be associated with +`*.guard.yaml` in a JSON Schema-aware editor. + +Format validation does not establish complete provider coverage, correct text +extraction, preservation of unrelated data, or equivalence to arbitrary Python +validators and coercion. These require implementation and source-aware tests. +Provider revision and digest checks are separate from document validation. + +The format is experimental: `1.0.0-alpha.1`, not stable version 1. Its version +is separate from the capability profile `single_text.v1` and provider API +revision. A profile identifies the supported content capability; it does not +choose deployment rails or enable runtime profile selection. + +This is a NeMo Guardrails document using JSON Schema vocabulary, not an OpenAPI +document or an OpenAPI Overlay. Generic schema tools do not execute its guardrail +annotations. diff --git a/nemoguardrails/server/experimental/contracts/guard-contract.schema.json b/nemoguardrails/server/experimental/contracts/guard-contract.schema.json new file mode 100644 index 0000000000..7e507a7f78 --- /dev/null +++ b/nemoguardrails/server/experimental/contracts/guard-contract.schema.json @@ -0,0 +1,510 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nemo-guardrails:guard-contract:1.0.0-alpha.1", + "title": "NeMo Guard Contract 1.0.0-alpha.1 (experimental)", + "description": "Guard contract document format. Schema validation checks syntax, not provider coverage or runtime capability.", + "type": "object", + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "version": { + "const": "1.0.0-alpha.1" + }, + "operationId": { + "type": "string", + "pattern": "\\S" + }, + "profile": { + "const": "single_text.v1" + }, + "request": { + "$ref": "#/$defs/schema" + }, + "response": { + "$ref": "#/$defs/schema" + }, + "stream": { + "allOf": [ + { + "$ref": "#/$defs/schema" + }, + { + "type": "object", + "required": [ + "oneOf", + "x-nemo-guardrails" + ], + "properties": { + "x-nemo-guardrails": { + "type": "object", + "required": [ + "transport" + ] + }, + "oneOf": { + "type": "array", + "items": { + "type": "object", + "required": [ + "x-nemo-guardrails" + ], + "properties": { + "x-nemo-guardrails": { + "type": "object", + "required": [ + "event" + ] + } + } + } + } + } + } + ] + }, + "excluded_stream_events": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string", + "pattern": "\\S" + }, + "sources": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "reason", + "sources" + ] + } + }, + "integration": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*$" + }, + "endpoint": { + "$ref": "#/$defs/endpoint" + } + }, + "required": [ + "endpoint" + ] + } + }, + "required": [ + "version", + "operationId", + "profile", + "request", + "response", + "integration" + ], + "dependentRequired": { + "excluded_stream_events": [ + "stream" + ] + }, + "$defs": { + "symbol": { + "type": "object", + "additionalProperties": false, + "properties": { + "module": { + "type": "string", + "pattern": "^[a-zA-Z_]\\w*(\\.[a-zA-Z_]\\w*)*$" + }, + "name": { + "type": "string", + "pattern": "^[a-zA-Z_]\\w*$" + } + }, + "required": [ + "module", + "name" + ] + }, + "endpoint": { + "type": "object", + "additionalProperties": false, + "properties": { + "route_path": { + "type": "string", + "pattern": "^/[^/].*[^/]$|^/[^/]$" + }, + "operation_label": { + "type": "string", + "pattern": "\\S" + }, + "unsupported_request_code": { + "type": "string", + "pattern": "\\S" + }, + "unsupported_response_code": { + "type": "string", + "pattern": "\\S" + }, + "stream_hooks": { + "$ref": "#/$defs/symbol" + }, + "api_revision": { + "$ref": "#/$defs/symbol" + } + }, + "required": [ + "unsupported_request_code", + "unsupported_response_code" + ] + }, + "subject": { + "type": "object", + "additionalProperties": false, + "properties": { + "kind": { + "const": "text" + }, + "role": { + "enum": [ + "user", + "assistant" + ] + }, + "replaceable": { + "type": "boolean" + }, + "replacement_blocked_by": { + "type": "string", + "minLength": 1 + }, + "replacement_reason": { + "type": "string", + "pattern": "^(core_capability|provider_integrity|projection_policy)\\.[a-z][a-z0-9_]*$" + } + }, + "required": [ + "kind", + "role" + ] + }, + "selection": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "integer", + "boolean" + ] + } + }, + "variant": { + "type": "object", + "additionalProperties": false, + "properties": { + "match": { + "$ref": "#/$defs/selection" + }, + "cardinality": { + "enum": [ + "any", + "at_most_one", + "exactly_one" + ] + } + } + }, + "eventVariant": { + "type": "object", + "additionalProperties": false, + "properties": { + "source_schema": { + "type": "string", + "minLength": 1 + }, + "shape": { + "type": "string", + "minLength": 1 + }, + "match": { + "$ref": "#/$defs/selection" + }, + "required_fields": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "external_reason": { + "type": "string", + "pattern": "^(core_capability|provider_integrity|projection_policy)\\.[a-z][a-z0-9_]*$" + } + }, + "required": [ + "source_schema", + "shape" + ] + }, + "event": { + "type": "object", + "additionalProperties": false, + "properties": { + "classification": { + "enum": [ + "guarded_delta", + "snapshot", + "opaque", + "provider_error" + ] + }, + "variants": { + "type": "array", + "items": { + "$ref": "#/$defs/eventVariant" + }, + "minItems": 1 + }, + "missing_text": { + "enum": [ + "reject", + "opaque", + "empty" + ] + }, + "missing_text_shape": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "classification", + "variants" + ] + }, + "transport": { + "type": "object", + "additionalProperties": false, + "properties": { + "data_discriminator": { + "type": "string", + "minLength": 1 + }, + "require_sse_event": { + "type": "boolean" + }, + "non_data_shape": { + "type": "string", + "minLength": 1 + }, + "sentinels": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1 + } + } + } + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "classification": { + "enum": [ + "guarded", + "constrained", + "opaque" + ] + }, + "subject": { + "$ref": "#/$defs/subject" + }, + "reason": { + "type": "string", + "pattern": "^(core_capability|provider_integrity|projection_policy)\\.[a-z][a-z0-9_]*$" + }, + "gate": { + "const": "disabled" + }, + "extension": { + "type": "boolean" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "pattern": "^#/components/schemas/[A-Za-z_][A-Za-z0-9._-]*$" + }, + "unknown_fields": { + "enum": [ + "configurable" + ] + }, + "stream_selector_field": { + "type": "string", + "minLength": 1 + }, + "opaque_fields": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "not": { + "const": "*" + } + }, + "minItems": 1, + "uniqueItems": true + }, + "variant": { + "$ref": "#/$defs/variant" + }, + "event": { + "$ref": "#/$defs/event" + }, + "transport": { + "$ref": "#/$defs/transport" + } + } + }, + "schema": { + "type": "object", + "additionalProperties": false, + "properties": { + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "$comment": { + "type": "string" + }, + "examples": { + "type": "array", + "items": {} + }, + "deprecated": { + "type": "boolean" + }, + "readOnly": { + "type": "boolean" + }, + "writeOnly": { + "type": "boolean" + }, + "type": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "const": {}, + "default": {}, + "enum": { + "type": "array", + "items": {}, + "minItems": 1, + "uniqueItems": true + }, + "minItems": { + "type": "integer", + "minimum": 0 + }, + "maxItems": { + "type": "integer", + "minimum": 0 + }, + "minLength": { + "type": "integer", + "minimum": 0 + }, + "maxLength": { + "type": "integer", + "minimum": 0 + }, + "pattern": { + "type": "string" + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/field" + } + }, + "required": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "additionalProperties": { + "type": "boolean" + }, + "items": { + "$ref": "#/$defs/schema" + }, + "oneOf": { + "type": "array", + "items": { + "$ref": "#/$defs/schema" + }, + "minItems": 1 + }, + "x-nemo-guardrails": { + "$ref": "#/$defs/metadata" + } + }, + "patternProperties": { + "^x-(?!nemo)": {} + } + }, + "field": { + "allOf": [ + { + "$ref": "#/$defs/schema" + }, + { + "type": "object", + "required": [ + "x-nemo-guardrails" + ], + "properties": { + "x-nemo-guardrails": { + "type": "object", + "required": [ + "classification" + ] + } + } + } + ] + } + } +} diff --git a/nemoguardrails/server/experimental/contracts/minimal.guard.example.yaml b/nemoguardrails/server/experimental/contracts/minimal.guard.example.yaml new file mode 100644 index 0000000000..a9e4409059 --- /dev/null +++ b/nemoguardrails/server/experimental/contracts/minimal.guard.example.yaml @@ -0,0 +1,40 @@ +version: '1.0.0-alpha.1' +operationId: createText +profile: single_text.v1 +request: + type: object + required: [prompt] + properties: + prompt: + type: string + minLength: 1 + x-nemo-guardrails: + classification: guarded + subject: + kind: text + role: user + replaceable: true + x-nemo-guardrails: + opaque_fields: [model] +response: + type: object + required: [text] + properties: + text: + type: string + minLength: 1 + x-nemo-guardrails: + classification: guarded + subject: + kind: text + role: assistant + replaceable: true + status: + const: complete + x-nemo-guardrails: + classification: constrained + reason: projection_policy.complete_text +integration: + endpoint: + unsupported_request_code: unsupported_text_request + unsupported_response_code: unsupported_text_response diff --git a/tests/server/experimental/test_guard_contract.py b/tests/server/experimental/test_guard_contract.py new file mode 100644 index 0000000000..4117394915 --- /dev/null +++ b/tests/server/experimental/test_guard_contract.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +from copy import deepcopy +from pathlib import Path + +import pytest +import yaml +from jsonschema import Draft202012Validator +from jsonschema.exceptions import ValidationError + +REPOSITORY_ROOT = Path(__file__).parents[3] +CONTRACTS_ROOT = REPOSITORY_ROOT / "nemoguardrails/server/experimental/contracts" +SCHEMA_PATH = CONTRACTS_ROOT / "guard-contract.schema.json" +MINIMAL_CONTRACT_PATH = CONTRACTS_ROOT / "minimal.guard.example.yaml" +MARKDOWN_LINK = re.compile(r"\[[^]]+\]\(([^)]+)\)") +EXTENSION = "x-nemo-guardrails" + + +@pytest.fixture(scope="module") +def guard_contract_schema() -> dict: + """Load the guard contract schema shared by the validation tests.""" + return json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def minimal_guard_contract() -> dict: + """Load the minimal guard contract example.""" + return yaml.safe_load(MINIMAL_CONTRACT_PATH.read_text(encoding="utf-8")) + + +@pytest.fixture +def streaming_guard_contract(minimal_guard_contract: dict) -> dict: + """Describe a text event and SSE sentinel without an endpoint implementation.""" + contract = deepcopy(minimal_guard_contract) + event = deepcopy(contract["response"]) + event[EXTENSION] = { + "event": { + "classification": "guarded_delta", + "variants": [ + { + "source_schema": "ExampleTextEvent", + "shape": "text.delta", + "required_fields": ["text"], + } + ], + "missing_text": "reject", + } + } + contract["stream"] = { + "oneOf": [event], + EXTENSION: { + "transport": { + "require_sse_event": False, + "non_data_shape": "[DONE]", + "sentinels": {"[DONE]": "[DONE]"}, + } + }, + } + return contract + + +def test_guard_contract_schema_is_valid(guard_contract_schema: dict) -> None: + """The guard contract schema is a valid JSON Schema 2020-12 document.""" + Draft202012Validator.check_schema(guard_contract_schema) + + +def test_minimal_guard_contract_matches_schema(guard_contract_schema: dict, minimal_guard_contract: dict) -> None: + """The minimal guard contract demonstrates a schema-valid document.""" + Draft202012Validator(guard_contract_schema).validate(minimal_guard_contract) + + +@pytest.mark.parametrize("projection", ["request", "response"]) +@pytest.mark.parametrize("payload", [None, True, 42, "text", []]) +def test_minimal_guard_projection_rejects_non_objects( + minimal_guard_contract: dict, projection: str, payload: object +) -> None: + """Both example projections require object payloads.""" + with pytest.raises(ValidationError): + Draft202012Validator(minimal_guard_contract[projection]).validate(payload) + + +@pytest.mark.parametrize(("projection", "field"), [("request", "prompt"), ("response", "text")]) +def test_minimal_guard_projection_requires_its_text_field( + minimal_guard_contract: dict, projection: str, field: str +) -> None: + """Object payloads pass only when the required text field is present.""" + validator = Draft202012Validator(minimal_guard_contract[projection]) + validator.validate({field: "hello"}) + + with pytest.raises(ValidationError, match="required property"): + validator.validate({}) + + +def test_guard_contract_rejects_unknown_version(guard_contract_schema: dict, minimal_guard_contract: dict) -> None: + """The schema rejects documents with an unsupported format version.""" + contract = minimal_guard_contract | {"version": "1.0.0"} + + with pytest.raises(ValidationError): + Draft202012Validator(guard_contract_schema).validate(contract) + + +def test_guard_contract_rejects_unknown_top_level_key( + guard_contract_schema: dict, minimal_guard_contract: dict +) -> None: + """The schema rejects misspelled or unsupported top-level keys.""" + contract = minimal_guard_contract | {"requests": {}} + + with pytest.raises(ValidationError): + Draft202012Validator(guard_contract_schema).validate(contract) + + +def test_guard_contract_allows_non_guardrails_schema_extensions( + guard_contract_schema: dict, minimal_guard_contract: dict +) -> None: + """Non-Guardrails schema extensions remain available to other tooling.""" + contract = deepcopy(minimal_guard_contract) + contract["request"]["x-provider-annotation"] = {"value": "example"} + + Draft202012Validator(guard_contract_schema).validate(contract) + + +@pytest.mark.parametrize("key", ["x-nemo-guardrail", "x-nemo-guardrails-extra", "x-nemoguardrails"]) +@pytest.mark.parametrize("on_field", [False, True]) +def test_guard_contract_rejects_guardrails_extension_lookalikes( + guard_contract_schema: dict, minimal_guard_contract: dict, key: str, on_field: bool +) -> None: + """The reserved x-nemo namespace rejects misspelled Guardrails keys on objects and fields.""" + contract = deepcopy(minimal_guard_contract) + target = contract["request"]["properties"]["prompt"] if on_field else contract["request"] + target[key] = {"classification": "opaque"} + + with pytest.raises(ValidationError): + Draft202012Validator(guard_contract_schema).validate(contract) + + +def test_stream_contract_matches_schema(guard_contract_schema: dict, streaming_guard_contract: dict) -> None: + """The format supports a stream section independently of runtime support.""" + Draft202012Validator(guard_contract_schema).validate(streaming_guard_contract) + + +@pytest.mark.parametrize("section", ["transport", "event"]) +def test_stream_contract_requires_framing_and_event_policy( + guard_contract_schema: dict, streaming_guard_contract: dict, section: str +) -> None: + """Stream roots need transport metadata and each family needs event policy.""" + stream = streaming_guard_contract["stream"] + metadata = stream[EXTENSION] if section == "transport" else stream["oneOf"][0][EXTENSION] + del metadata[section] + + with pytest.raises(ValidationError): + Draft202012Validator(guard_contract_schema).validate(streaming_guard_contract) + + +@pytest.mark.parametrize( + ("key", "value"), + [ + ("event_schema_binding", "operation_response"), + ("event_schema_reason", "provider_integrity.component_source"), + ], +) +def test_stream_contract_rejects_source_resolution_settings( + guard_contract_schema: dict, streaming_guard_contract: dict, key: str, value: str +) -> None: + """Source resolution settings are not part of the exported policy vocabulary.""" + streaming_guard_contract["stream"][EXTENSION][key] = value + + with pytest.raises(ValidationError): + Draft202012Validator(guard_contract_schema).validate(streaming_guard_contract) + + +def test_contract_documentation_has_no_broken_local_links() -> None: + """Relative links in the contract documentation resolve locally.""" + broken_links: list[str] = [] + + for document in CONTRACTS_ROOT.rglob("*.md"): + for link in MARKDOWN_LINK.findall(document.read_text(encoding="utf-8")): + if "://" in link or link.startswith("#"): + continue + target, _, _fragment = link.partition("#") + if target and not (document.parent / target).exists(): + broken_links.append(f"{document.name}: {link}") + + assert broken_links == []