From ac429f613097d8c05bfccae04a88e69929c656ae Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:17:31 +0200 Subject: [PATCH 01/13] feat(server): integrate OpenAI Chat streaming Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/_buffered_kernel.py | 60 ++- .../server/experimental/_guarded_proxy.py | 74 ++- .../server/experimental/_guarded_stream.py | 8 + .../server/experimental/_http_kernel.py | 29 +- .../server/experimental/_streaming_http.py | 7 +- .../server/experimental/provider/endpoint.py | 10 + .../server/experimental/provider/errors.py | 8 +- .../openai/chat_completions/endpoint.py | 11 + .../experimental/providers/openai/errors.py | 29 + .../providers/openai/integration.py | 16 +- .../experimental/test_contract_export.py | 12 +- .../test_openai_chat_integration.py | 13 +- .../test_openai_chat_streaming_integration.py | 497 ++++++++++++++++++ .../experimental/test_streaming_http.py | 13 +- 14 files changed, 754 insertions(+), 33 deletions(-) create mode 100644 tests/server/experimental/test_openai_chat_streaming_integration.py diff --git a/nemoguardrails/server/experimental/_buffered_kernel.py b/nemoguardrails/server/experimental/_buffered_kernel.py index e912f257a2..15f10df4cd 100644 --- a/nemoguardrails/server/experimental/_buffered_kernel.py +++ b/nemoguardrails/server/experimental/_buffered_kernel.py @@ -91,6 +91,15 @@ class OperationProjectionFailed: failure: UnsupportedGuardedPayload +@dataclass(frozen=True, slots=True) +class PreparedOperationInput(Generic[RequestT]): + """Carry one projected and inspected input into its response mode.""" + + request: RequestT + input_message: GuardedMessage + resolved: _ResolvedContentChecker + + def _project_message( projection: GuardedMessageProjection[PayloadT], payload: PayloadT, @@ -146,19 +155,18 @@ async def _run_check( return _stopped_operation(stage, decision) -async def execute_buffered_operation( +async def prepare_operation_input( operation: BufferedGuardedOperation[RequestT, ResponseT], checker: ContentChecker | _ResolvedContentChecker, request: RequestT, - dispatch: Callable[[RequestT], Awaitable[ResponseT]], ) -> ( - OperationCompleted[ResponseT] + PreparedOperationInput[RequestT] | OperationBlocked | OperationCheckFailed | OperationModificationUnsupported | OperationProjectionFailed ): - """Execute one buffered operation with exactly one statically bound checker.""" + """Project and inspect one operation input exactly once.""" validated = checker if isinstance(checker, _ResolvedContentChecker) else validate_content_checker(checker) validated_checker = validated.checker @@ -178,9 +186,25 @@ async def execute_buffered_operation( if stopped is not None: return stopped - response = await dispatch(request) + return PreparedOperationInput(request, input_message, validated) + + +async def execute_prepared_buffered_operation( + operation: BufferedGuardedOperation[RequestT, ResponseT], + prepared: PreparedOperationInput[RequestT], + dispatch: Callable[[RequestT], Awaitable[ResponseT]], +) -> ( + OperationCompleted[ResponseT] + | OperationBlocked + | OperationCheckFailed + | OperationModificationUnsupported + | OperationProjectionFailed +): + """Dispatch and inspect a buffered response for one prepared input.""" + + response = await dispatch(prepared.request) - if policy.inspect_output: + if prepared.resolved.policy.inspect_output: try: output_message = _project_message(operation.output_projection, response, "assistant") except UnsupportedGuardedPayload as failure: @@ -189,9 +213,31 @@ async def execute_buffered_operation( return OperationCompleted(response) stopped = await _run_check( InspectionStage.OUTPUT, - lambda: validated_checker.check_output(OutputContentCheck(input_message, output_message.content)), + lambda: prepared.resolved.checker.check_output( + OutputContentCheck(prepared.input_message, output_message.content) + ), ) if stopped is not None: return stopped return OperationCompleted(response) + + +async def execute_buffered_operation( + operation: BufferedGuardedOperation[RequestT, ResponseT], + checker: ContentChecker | _ResolvedContentChecker, + request: RequestT, + dispatch: Callable[[RequestT], Awaitable[ResponseT]], +) -> ( + OperationCompleted[ResponseT] + | OperationBlocked + | OperationCheckFailed + | OperationModificationUnsupported + | OperationProjectionFailed +): + """Execute one buffered operation with exactly one statically bound checker.""" + + prepared = await prepare_operation_input(operation, checker, request) + if not isinstance(prepared, PreparedOperationInput): + return prepared + return await execute_prepared_buffered_operation(operation, prepared, dispatch) diff --git a/nemoguardrails/server/experimental/_guarded_proxy.py b/nemoguardrails/server/experimental/_guarded_proxy.py index a59325f907..547517baab 100644 --- a/nemoguardrails/server/experimental/_guarded_proxy.py +++ b/nemoguardrails/server/experimental/_guarded_proxy.py @@ -13,12 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Compose provider declarations with the guarded buffered HTTP pipeline.""" +"""Compose provider declarations with the guarded HTTP pipeline.""" from dataclasses import dataclass from pydantic import ValidationError +from nemoguardrails.server.experimental._buffered_kernel import ( + InspectionStage, + OperationProjectionFailed, + PreparedOperationInput, +) from nemoguardrails.server.experimental._guarded_operation import ( BufferedGuardedOperation, ContentInspectionNotApplicable, @@ -26,6 +31,10 @@ UnsupportedGuardedPayload, UnsupportedGuardedRepresentation, ) +from nemoguardrails.server.experimental._guarded_stream import ( + StreamInspectionUnsupported, + UnsupportedStreamInspection, +) from nemoguardrails.server.experimental._http_kernel import ( BufferedHttpRequest, BufferedHttpResponse, @@ -35,6 +44,7 @@ _has_media_type, ) from nemoguardrails.server.experimental._json_payload import InvalidJson, UnsupportedJsonShape, parse_json_object +from nemoguardrails.server.experimental._streaming_http import StreamingHttpDispatch, execute_streaming_http from nemoguardrails.server.experimental.provider.endpoint import GuardedJsonEndpoint from nemoguardrails.server.experimental.provider.errors import ProviderErrorMapping from nemoguardrails.server.experimental.provider.payload import GuardedMessageTarget, guarded_schema_error @@ -49,6 +59,7 @@ class GuardableProviderRequest: request: BufferedHttpRequest target: GuardedMessageTarget + streaming: bool = False def _require_json_representation(request: BufferedHttpRequest) -> None: @@ -89,21 +100,27 @@ def _prepare_guardable_request( raise UnsupportedGuardedPayload( guarded_schema_error(error, "request"), endpoint.unsupported_request_code ) from error - if projection.streams_response: + streaming = projection.streams_response + if streaming and endpoint.stream_adapter_factory is None: raise UnsupportedGuardedPayload( - "The guarded endpoint does not support streaming responses yet.", endpoint.unsupported_request_code + "The guarded endpoint does not support streaming responses.", endpoint.unsupported_request_code ) return GuardableProviderRequest( request=request, target=target, + streaming=streaming, ) -def create_buffered_guarded_http_operation( +def create_guarded_http_operation( endpoint: GuardedJsonEndpoint, errors: ProviderErrorMapping, + *, + stream_dispatch: StreamingHttpDispatch | None = None, + max_stream_event_bytes: int, + max_pending_stream_bytes: int, ) -> GuardedHttpOperation: - """Create one buffered HTTP operation from the provider endpoint contract.""" + """Create one buffered-and-streaming operation from an endpoint contract.""" def prepare_request(request: BufferedHttpRequest) -> GuardableProviderRequest: """Prepare one buffered HTTP request for guarded execution.""" @@ -153,6 +170,50 @@ def project_response( if issubclass(request_model, PolicyModel) else request_model.model_json_schema() ) + + async def handle_prepared_request( + prepared_input: PreparedOperationInput[GuardableProviderRequest], + ): + guardable = prepared_input.request + if not guardable.streaming: + return None + if stream_dispatch is None: + return errors.renderer( + OperationProjectionFailed( + InspectionStage.INPUT, + UnsupportedGuardedPayload("The guarded endpoint does not have streaming dispatch configured."), + ) + ) + streaming_policy = prepared_input.resolved.policy.stream_buffering + if prepared_input.resolved.policy.inspect_output and streaming_policy is None: + return errors.renderer( + StreamInspectionUnsupported( + UnsupportedStreamInspection("Guarded output streaming requires a buffering policy.") + ) + ) + adapter_factory = endpoint.stream_adapter_factory + if adapter_factory is None: + raise AssertionError("A streaming request requires an endpoint stream adapter.") + return await execute_streaming_http( + guardable.request, + dispatch=stream_dispatch, + checker=prepared_input.resolved.checker, + streaming_policy=streaming_policy, + input_message=guardable.target.message, + adapter=adapter_factory(), + render_outcome=errors.renderer, + max_event_bytes=max_stream_event_bytes, + max_pending_bytes=max_pending_stream_bytes, + ) + + success_content = {"application/json": {"schema": {}}} + if endpoint.stream_adapter_factory is not None: + success_content["text/event-stream"] = {"schema": {"type": "string"}} + documented_responses = errors.documented_responses + documented_responses[200] = { + "description": "Provider-native successful response.", + "content": success_content, + } openapi_extra = { "requestBody": { "required": True, @@ -177,7 +238,8 @@ def project_response( operation=operation, prepare_request=prepare_request, forward_request=forward_request, - documented_responses=errors.documented_responses, + prepared_request_handler=handle_prepared_request, + documented_responses=documented_responses, guarded_operation_paths=endpoint.operation_paths, openapi_extra=openapi_extra, ) diff --git a/nemoguardrails/server/experimental/_guarded_stream.py b/nemoguardrails/server/experimental/_guarded_stream.py index bfd717d1b0..1263c9b2ee 100644 --- a/nemoguardrails/server/experimental/_guarded_stream.py +++ b/nemoguardrails/server/experimental/_guarded_stream.py @@ -83,11 +83,19 @@ class StreamProcessingFailed: failure: BaseException +@dataclass(frozen=True, slots=True) +class StreamInspectionUnsupported: + """Stop before dispatch when output inspection cannot guard a stream.""" + + failure: UnsupportedStreamInspection + + StreamOutcome = ( OperationBlocked | OperationCheckFailed | OperationModificationUnsupported | OperationProjectionFailed + | StreamInspectionUnsupported | StreamUpstreamFailed | StreamProcessingFailed ) diff --git a/nemoguardrails/server/experimental/_http_kernel.py b/nemoguardrails/server/experimental/_http_kernel.py index a6c1e36cca..31257a5d95 100644 --- a/nemoguardrails/server/experimental/_http_kernel.py +++ b/nemoguardrails/server/experimental/_http_kernel.py @@ -34,7 +34,9 @@ OperationCompleted, OperationModificationUnsupported, OperationProjectionFailed, - execute_buffered_operation, + PreparedOperationInput, + execute_prepared_buffered_operation, + prepare_operation_input, ) from nemoguardrails.server.experimental._content_checker import ( ContentChecker, @@ -148,12 +150,20 @@ def __post_init__(self) -> None: @dataclass(frozen=True, slots=True) class GuardedHttpOperation: - """Bind one buffered guarded operation to an owned HTTP path.""" + """Bind one guarded operation to an owned HTTP path. + + The prepared-request handler is the private response-mode seam used while + buffered and streaming transports remain separately injected. It runs only + after the shared input projection and check have completed. + """ operation_path: GuardedOperationPath operation: BufferedGuardedOperation[Any, BufferedHttpResponse] prepare_request: Callable[[BufferedHttpRequest], Any] | None = None forward_request: Callable[[Any], BufferedHttpRequest] | None = None + prepared_request_handler: ( + Callable[[PreparedOperationInput[Any]], Awaitable[BufferedHttpResponse | Response | None]] | None + ) = None documented_responses: dict[int | str, dict[str, Any]] | None = None guarded_operation_paths: tuple[GuardedOperationPath, ...] = () openapi_extra: dict[str, Any] | None = None @@ -510,11 +520,24 @@ async def handle(request: Request) -> Response: return _render_failure( OperationProjectionFailed(InspectionStage.INPUT, failure), render_outcome, + request.method, ) - outcome = await execute_buffered_operation( + prepared = await prepare_operation_input( declaration.operation, checker, operation_request, + ) + if not isinstance(prepared, PreparedOperationInput): + return _render_failure(prepared, render_outcome, request.method) + if declaration.prepared_request_handler is not None: + response = await declaration.prepared_request_handler(prepared) + if isinstance(response, BufferedHttpResponse): + return _render_response(response, request.method) + if response is not None: + return response + outcome = await execute_prepared_buffered_operation( + declaration.operation, + prepared, bounded_dispatch, ) except RequestBodyTooLarge as failure: diff --git a/nemoguardrails/server/experimental/_streaming_http.py b/nemoguardrails/server/experimental/_streaming_http.py index 8ce765299f..a3665c4513 100644 --- a/nemoguardrails/server/experimental/_streaming_http.py +++ b/nemoguardrails/server/experimental/_streaming_http.py @@ -27,9 +27,11 @@ from nemoguardrails.server.experimental._content_checker import ContentChecker, StreamBufferingPolicy from nemoguardrails.server.experimental._guarded_operation import UnsupportedGuardedPayload from nemoguardrails.server.experimental._guarded_stream import ( + StreamInspectionUnsupported, StreamOutcomeRenderer, StreamProcessingFailed, StreamUpstreamFailed, + UnsupportedStreamInspection, guard_provider_stream, validate_streaming_policy, ) @@ -180,7 +182,10 @@ async def execute_streaming_http( if input_message.role != "user": raise ValueError("A guarded stream requires a user input message.") if streaming_policy is not None: - validate_streaming_policy(streaming_policy) + try: + validate_streaming_policy(streaming_policy) + except UnsupportedStreamInspection as failure: + return _render_response(render_outcome(StreamInspectionUnsupported(failure)), request.method) # An encoded stream cannot be inspected; uninspected relays keep the client's preference. request = replace(request, headers=_request_identity_encoding(request.headers)) diff --git a/nemoguardrails/server/experimental/provider/endpoint.py b/nemoguardrails/server/experimental/provider/endpoint.py index 2f31cda6c4..845ea747e9 100644 --- a/nemoguardrails/server/experimental/provider/endpoint.py +++ b/nemoguardrails/server/experimental/provider/endpoint.py @@ -16,6 +16,7 @@ """Declare provider endpoints handled by the guarded JSON pipeline.""" import re +from collections.abc import Callable from dataclasses import dataclass from nemoguardrails.server.experimental._http_paths import GuardedOperationPath @@ -24,6 +25,7 @@ GuardedRequestModel, validate_payload_projection_contract, ) +from nemoguardrails.server.experimental.provider.stream import ProviderStreamAdapter, StreamProjectionContract from nemoguardrails.server.experimental.provider.transport import ProviderApiRevisionBinding @@ -42,6 +44,7 @@ class GuardedJsonEndpoint: contract_name: str | None = None method: str = "POST" api_revision: ProviderApiRevisionBinding | None = None + stream_adapter_factory: Callable[[], ProviderStreamAdapter] | None = None operation_paths: tuple[GuardedOperationPath, ...] = () def __post_init__(self) -> None: @@ -75,3 +78,10 @@ def __post_init__(self) -> None: response_contract = validate_payload_projection_contract(self.guarded_response_model, "response") if request_contract.profile != response_contract.profile: raise ValueError("Guarded request and response capability profiles must match.") + if self.stream_adapter_factory is not None: + adapter = self.stream_adapter_factory() + required_methods = ("classify_event", "validate_end_of_stream", "encode_error") + if any(not callable(getattr(adapter, method, None)) for method in required_methods): + raise ValueError("The stream adapter factory must create a provider stream adapter.") + if not isinstance(getattr(adapter, "contract", None), StreamProjectionContract): + raise ValueError("The stream adapter must declare its stream projection contract.") diff --git a/nemoguardrails/server/experimental/provider/errors.py b/nemoguardrails/server/experimental/provider/errors.py index 0c971f371a..0880e7e214 100644 --- a/nemoguardrails/server/experimental/provider/errors.py +++ b/nemoguardrails/server/experimental/provider/errors.py @@ -15,12 +15,16 @@ """Bind provider-native runtime errors to their documented response models.""" +from collections.abc import Callable from dataclasses import dataclass from typing import Any from pydantic import BaseModel -from nemoguardrails.server.experimental._http_kernel import OutcomeRenderer +from nemoguardrails.server.experimental._guarded_stream import StreamOutcome +from nemoguardrails.server.experimental._http_kernel import BufferedHttpResponse, HttpOperationFailed, HttpRouteRejected + +ProviderOutcomeRenderer = Callable[[StreamOutcome | HttpOperationFailed | HttpRouteRejected], BufferedHttpResponse] @dataclass(frozen=True, slots=True) @@ -36,7 +40,7 @@ class ProviderErrorResponse: class ProviderErrorMapping: """Make one provider error authority serve runtime and OpenAPI.""" - renderer: OutcomeRenderer + renderer: ProviderOutcomeRenderer responses: tuple[ProviderErrorResponse, ...] def __post_init__(self) -> None: diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py b/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py index 7cb88dd80e..5ca35e8d73 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py @@ -17,12 +17,19 @@ from nemoguardrails.server.experimental._http_paths import GuardedOperationPath from nemoguardrails.server.experimental.provider.endpoint import GuardedJsonEndpoint +from nemoguardrails.server.experimental.provider.stream import create_classified_stream_adapter_factory from nemoguardrails.server.experimental.providers.openai.chat_completions.request_binding import ( ChatCompletionsGuardedRequest, ) from nemoguardrails.server.experimental.providers.openai.chat_completions.response_binding import ( ChatCompletionsGuardedResponse, ) +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_classifier import ( + STREAM_CLASSIFIER, +) +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_hooks import ( + ChatCompletionsStreamHooks, +) CHAT_COMPLETIONS_ENDPOINT = GuardedJsonEndpoint( provider_operation_id="createChatCompletion", @@ -41,4 +48,8 @@ frozenset({"POST"}), ), ), + stream_adapter_factory=create_classified_stream_adapter_factory( + STREAM_CLASSIFIER, + ChatCompletionsStreamHooks, + ), ) diff --git a/nemoguardrails/server/experimental/providers/openai/errors.py b/nemoguardrails/server/experimental/providers/openai/errors.py index e54621eed0..8d0041f384 100644 --- a/nemoguardrails/server/experimental/providers/openai/errors.py +++ b/nemoguardrails/server/experimental/providers/openai/errors.py @@ -30,6 +30,11 @@ InvalidGuardedPayload, UnsupportedGuardedRepresentation, ) +from nemoguardrails.server.experimental._guarded_stream import ( + StreamInspectionUnsupported, + StreamProcessingFailed, + StreamUpstreamFailed, +) from nemoguardrails.server.experimental._http_kernel import ( BufferedHttpResponse, HttpFailureKind, @@ -85,6 +90,9 @@ def render_openai_error( | OperationProjectionFailed | HttpOperationFailed | HttpRouteRejected + | StreamInspectionUnsupported + | StreamProcessingFailed + | StreamUpstreamFailed ), ) -> BufferedHttpResponse: """Render one typed proxy outcome with the OpenAI-compatible envelope.""" @@ -133,6 +141,27 @@ def render_openai_error( "unsupported_response", "response_content_not_replaceable", ) + if isinstance(outcome, StreamInspectionUnsupported): + return _error_response( + 422, + "The configured output inspection policy cannot guard a streamed response.", + "unsupported_request", + "unsupported_stream_inspection_policy", + ) + if isinstance(outcome, StreamUpstreamFailed): + return _error_response( + 502, + "The upstream OpenAI Chat Completions stream failed.", + "proxy_error", + "upstream_stream_failed", + ) + if isinstance(outcome, StreamProcessingFailed): + return _error_response( + 502, + "The OpenAI Chat Completions stream could not be processed safely.", + "proxy_error", + "stream_processing_failed", + ) if isinstance(outcome, HttpOperationFailed): status_code, error_type, message = { HttpFailureKind.INVALID_CONTENT_LENGTH: (400, "invalid_request_error", "Invalid Content-Length."), diff --git a/nemoguardrails/server/experimental/providers/openai/integration.py b/nemoguardrails/server/experimental/providers/openai/integration.py index af9d2afa60..a0dcd9a3cc 100644 --- a/nemoguardrails/server/experimental/providers/openai/integration.py +++ b/nemoguardrails/server/experimental/providers/openai/integration.py @@ -20,13 +20,14 @@ from fastapi import APIRouter from nemoguardrails.server.experimental._content_checker import ContentChecker -from nemoguardrails.server.experimental._guarded_proxy import create_buffered_guarded_http_operation +from nemoguardrails.server.experimental._guarded_proxy import create_guarded_http_operation from nemoguardrails.server.experimental._http_kernel import ( DEFAULT_MAX_REQUEST_BODY_BYTES, DEFAULT_MAX_RESPONSE_BODY_BYTES, HttpDispatch, create_http_proxy_router, ) +from nemoguardrails.server.experimental._streaming_http import StreamingHttpDispatch from nemoguardrails.server.experimental.providers.openai.chat_completions.endpoint import ( CHAT_COMPLETIONS_ENDPOINT, ) @@ -37,14 +38,23 @@ def create_openai_chat_router( *, checker: ContentChecker, dispatch: HttpDispatch, + stream_dispatch: StreamingHttpDispatch | None = None, reserved_routes: Mapping[str, Collection[str]] | None = None, max_request_body_bytes: int = DEFAULT_MAX_REQUEST_BODY_BYTES, max_response_body_bytes: int = DEFAULT_MAX_RESPONSE_BODY_BYTES, ) -> APIRouter: - """Create the private buffered OpenAI Chat proxy surface.""" + """Create the private buffered and streaming OpenAI Chat proxy surface.""" return create_http_proxy_router( - operations=(create_buffered_guarded_http_operation(CHAT_COMPLETIONS_ENDPOINT, OPENAI_ERROR_MAPPING),), + operations=( + create_guarded_http_operation( + CHAT_COMPLETIONS_ENDPOINT, + OPENAI_ERROR_MAPPING, + stream_dispatch=stream_dispatch, + max_stream_event_bytes=max_response_body_bytes, + max_pending_stream_bytes=max_response_body_bytes, + ), + ), checker=checker, dispatch=dispatch, render_outcome=OPENAI_ERROR_MAPPING.renderer, diff --git a/tests/server/experimental/test_contract_export.py b/tests/server/experimental/test_contract_export.py index 451f71e337..2c9f3bb276 100644 --- a/tests/server/experimental/test_contract_export.py +++ b/tests/server/experimental/test_contract_export.py @@ -193,7 +193,7 @@ def test_export_follows_changed_endpoint_metadata(example_endpoint): def test_http_schema_does_not_require_exportable_unions(example_endpoint): """Policy-aware HTTP schemas preserve unions outside the contract format.""" - from nemoguardrails.server.experimental._guarded_proxy import create_buffered_guarded_http_operation + from nemoguardrails.server.experimental._guarded_proxy import create_guarded_http_operation from nemoguardrails.server.experimental.provider.projection_policy import constrained from nemoguardrails.server.experimental.providers.openai.errors import OPENAI_ERROR_MAPPING @@ -214,7 +214,9 @@ def streams_response(self) -> bool: with pytest.raises(ValueError, match="disjoint nullable"): export_guard_contract(endpoint) - operation = create_buffered_guarded_http_operation(endpoint, OPENAI_ERROR_MAPPING) + operation = create_guarded_http_operation( + endpoint, OPENAI_ERROR_MAPPING, max_stream_event_bytes=1024, max_pending_stream_bytes=1024 + ) schema = operation.openapi_extra["requestBody"]["content"]["application/json"]["schema"] assert schema["additionalProperties"] is False assert schema["properties"]["option"]["anyOf"] == [{"type": "string"}, {"type": "integer"}] @@ -226,7 +228,7 @@ def streams_response(self) -> bool: def test_http_schema_supports_models_without_policy_annotations(example_endpoint): """Handwritten bindings need not support contract export to construct a route.""" - from nemoguardrails.server.experimental._guarded_proxy import create_buffered_guarded_http_operation + from nemoguardrails.server.experimental._guarded_proxy import create_guarded_http_operation from nemoguardrails.server.experimental.providers.openai.errors import OPENAI_ERROR_MAPPING class PlainRequest(GuardedRequestModel): @@ -240,7 +242,9 @@ def streams_response(self) -> bool: PlainRequest.guarded_text_location = example_endpoint.guarded_request_model.guarded_text_location endpoint = replace(example_endpoint, guarded_request_model=PlainRequest) - operation = create_buffered_guarded_http_operation(endpoint, OPENAI_ERROR_MAPPING) + operation = create_guarded_http_operation( + endpoint, OPENAI_ERROR_MAPPING, max_stream_event_bytes=1024, max_pending_stream_bytes=1024 + ) assert operation.openapi_extra["requestBody"]["content"]["application/json"]["schema"] == ( PlainRequest.model_json_schema() diff --git a/tests/server/experimental/test_openai_chat_integration.py b/tests/server/experimental/test_openai_chat_integration.py index c3c02e3191..8191c12526 100644 --- a/tests/server/experimental/test_openai_chat_integration.py +++ b/tests/server/experimental/test_openai_chat_integration.py @@ -32,7 +32,7 @@ ContentInspectionPolicy, ) from nemoguardrails.server.experimental._guarded_operation import UnsupportedGuardedPayload -from nemoguardrails.server.experimental._guarded_proxy import create_buffered_guarded_http_operation +from nemoguardrails.server.experimental._guarded_proxy import create_guarded_http_operation from nemoguardrails.server.experimental._http_kernel import ( BufferedHttpRequest, BufferedHttpResponse, @@ -92,6 +92,11 @@ def _response_body(): return b'{ "id" : "chatcmpl-example", "choices" : [ { "index" : 0, "message" : { "role" : "assistant", "content" : "answer" } } ], "usage" : { "total_tokens" : 2 } }' +def _guarded_operation(endpoint, errors): + """Build the guarded operation with default stream limits.""" + return create_guarded_http_operation(endpoint, errors, max_stream_event_bytes=1024, max_pending_stream_bytes=1024) + + def _json_headers(*headers): """Return JSON content headers with optional provider metadata.""" return ((b"content-type", b"application/json"), *headers) @@ -118,7 +123,7 @@ def test_buffered_request_preparation_preserves_the_complete_provider_request(): headers=_json_headers((b"x-provider-opaque", b"first"), (b"x-provider-opaque", b"second")), body=_request_body(), ) - operation = create_buffered_guarded_http_operation(CHAT_COMPLETIONS_ENDPOINT, OPENAI_ERROR_MAPPING) + operation = _guarded_operation(CHAT_COMPLETIONS_ENDPOINT, OPENAI_ERROR_MAPPING) prepared = operation.prepare_request(request) @@ -620,7 +625,7 @@ async def dispatch(request): app = FastAPI() app.include_router( create_http_proxy_router( - operations=(create_buffered_guarded_http_operation(endpoint, OPENAI_ERROR_MAPPING),), + operations=(_guarded_operation(endpoint, OPENAI_ERROR_MAPPING),), checker=checker, dispatch=dispatch, render_outcome=OPENAI_ERROR_MAPPING.renderer, @@ -776,7 +781,7 @@ def _revision_bound_operation(): transport_name="provider-version", ) endpoint = replace(CHAT_COMPLETIONS_ENDPOINT, api_revision=revision) - return create_buffered_guarded_http_operation(endpoint, OPENAI_ERROR_MAPPING) + return _guarded_operation(endpoint, OPENAI_ERROR_MAPPING) def _chat_request(*headers): diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py new file mode 100644 index 0000000000..82367dc594 --- /dev/null +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -0,0 +1,497 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import httpx +import pytest +from fastapi import FastAPI + +from nemoguardrails.server.experimental._content_checker import ( + ContentAllowed, + ContentBlocked, + ContentInspectionPolicy, + StreamBufferingPolicy, +) +from nemoguardrails.server.experimental._http_kernel import BufferedHttpResponse +from nemoguardrails.server.experimental._streaming_http import StreamingHttpResponse +from nemoguardrails.server.experimental.providers.openai.chat_completions.endpoint import ( + CHAT_COMPLETIONS_ENDPOINT, +) +from nemoguardrails.server.experimental.providers.openai.errors import OPENAI_ERROR_MAPPING +from nemoguardrails.server.experimental.providers.openai.integration import create_openai_chat_router + + +class StaticChecker: + def __init__(self, policy, input_decision=ContentAllowed(), output_decisions=()): + self.policy = policy + self.input_decision = input_decision + self.output_decisions = list(output_decisions) + self.calls = [] + self.policy_reads = 0 + + def inspection_policy(self): + self.policy_reads += 1 + return self.policy + + async def check_input(self, check): + self.calls.append(("input", check)) + return self.input_decision + + async def check_output(self, check): + self.calls.append(("output", check)) + return self.output_decisions.pop(0) if self.output_decisions else ContentAllowed() + + +class Source: + def __init__(self, chunks, failure=None): + self.chunks = iter(chunks) + self.failure = failure + self.closed = False + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self.chunks) + except StopIteration: + if self.failure is not None: + failure, self.failure = self.failure, None + raise failure + raise StopAsyncIteration + + async def aclose(self): + self.closed = True + + +def request_body(**updates): + payload = { + "model": "gpt-example", + "messages": [{"role": "user", "content": "question"}], + "stream": True, + } + payload.update(updates) + return json.dumps(payload, separators=(",", ":")).encode() + + +def stream_event(content=None, **delta_updates): + delta = {"content": content} if content is not None else {"role": "assistant"} + delta.update(delta_updates) + payload = { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": delta, "finish_reason": None}], + } + return b"data: " + json.dumps(payload, separators=(",", ":")).encode() + b"\n\n" + + +def make_client(checker, stream_dispatch=None, buffered_dispatch=None, max_response_body_bytes=10 * 1024 * 1024): + async def reject_buffered(_request): + raise AssertionError("a selected streaming request must not use buffered dispatch") + + app = FastAPI() + app.include_router( + create_openai_chat_router( + checker=checker, + dispatch=buffered_dispatch or reject_buffered, + stream_dispatch=stream_dispatch, + max_response_body_bytes=max_response_body_bytes, + ) + ) + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://proxy.test") + + +async def post_stream(client, body=None, **kwargs): + headers = {"content-type": "application/json", **kwargs.pop("headers", {})} + return await client.post("/v1/chat/completions", content=body or request_body(), headers=headers, **kwargs) + + +@pytest.mark.asyncio +async def test_streaming_chat_uses_one_prepared_request_and_preserves_provider_sse_bytes(): + events = [ + stream_event(), + stream_event("safe "), + stream_event("streamed "), + stream_event("provider "), + stream_event("response"), + b"data: [DONE]\n\n", + ] + stream = b"".join(events) + source = Source([stream[:31], stream[31:109], stream[109:]]) + dispatched = [] + + async def stream_dispatch(request): + dispatched.append(request) + return StreamingHttpResponse( + 200, + ( + (b"content-type", b"text/event-stream; charset=utf-8"), + (b"content-length", str(len(stream)).encode()), + (b"x-request-id", b"provider-stream-id"), + ), + source, + ) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(2, 0))) + body = request_body() + async with make_client(checker, stream_dispatch) as client: + response = await post_stream( + client, + body, + params={"provider": "opaque"}, + headers={"x-provider-option": "preserve"}, + ) + + assert response.status_code == 200 + assert response.content == stream + assert response.headers["x-request-id"] == "provider-stream-id" + assert "content-length" not in response.headers + assert dispatched[0].body == body + assert dispatched[0].query == b"provider=opaque" + assert (b"x-provider-option", b"preserve") in dispatched[0].headers + assert [kind for kind, _ in checker.calls] == ["input", "output", "output"] + assert [call.output_content for kind, call in checker.calls if kind == "output"] == [ + "safe streamed ", + "provider response", + ] + assert checker.policy_reads == 1 + assert source.closed is True + + +@pytest.mark.asyncio +async def test_non_streaming_request_uses_the_same_operation_and_buffered_dispatch(): + buffered = [] + streamed = [] + + async def buffered_dispatch(request): + buffered.append(request) + return BufferedHttpResponse( + 200, + ((b"content-type", b"application/json"),), + b'{"choices":[{"index":0,"message":{"role":"assistant","content":"answer"}}]}', + ) + + async def stream_dispatch(request): + streamed.append(request) + raise AssertionError("a buffered request must not use streaming dispatch") + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch, buffered_dispatch) as client: + response = await post_stream(client, request_body(stream=False)) + + assert response.status_code == 200 + assert len(buffered) == 1 + assert streamed == [] + assert [kind for kind, _ in checker.calls] == ["input", "output"] + assert checker.policy_reads == 1 + + +@pytest.mark.asyncio +async def test_streaming_input_block_and_unsupported_policy_stop_before_dispatch(): + dispatched = [] + + async def stream_dispatch(request): + dispatched.append(request) + raise AssertionError("a stopped request must not be dispatched") + + blocked = StaticChecker( + ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0)), + input_decision=ContentBlocked("blocked", "policy"), + ) + async with make_client(blocked, stream_dispatch) as client: + blocked_response = await post_stream(client) + + unsupported = StaticChecker(ContentInspectionPolicy(True, True)) + async with make_client(unsupported, stream_dispatch) as client: + unsupported_response = await post_stream(client) + + assert blocked_response.status_code == 400 + assert blocked_response.json()["error"]["code"] == "content_blocked" + assert blocked_response.json()["error"]["param"] == "policy" + assert unsupported_response.status_code == 422 + assert unsupported_response.json()["error"]["code"] == "unsupported_stream_inspection_policy" + assert dispatched == [] + + +@pytest.mark.asyncio +async def test_missing_stream_dispatch_does_not_fall_through_to_buffered_dispatch(): + buffered = [] + + async def buffered_dispatch(request): + buffered.append(request) + raise AssertionError("a streaming request must not fall through to buffered dispatch") + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, buffered_dispatch=buffered_dispatch) as client: + response = await post_stream(client) + + assert response.status_code == 422 + assert response.json()["error"]["code"] == "unsupported_chat_completions_shape" + assert buffered == [] + + +@pytest.mark.asyncio +async def test_unsupported_stream_request_stops_before_stream_dispatch(): + dispatched = [] + + async def stream_dispatch(request): + dispatched.append(request) + raise AssertionError("an unsupported request must not be dispatched") + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client, request_body(tools=[{"type": "function"}])) + + assert response.status_code == 422 + assert response.json()["error"]["code"] == "unsupported_chat_completions_shape" + assert dispatched == [] + + +@pytest.mark.asyncio +async def test_streaming_output_block_hides_the_pending_provider_window(): + unsafe = stream_event("INTERNAL-") + stream_event("SECRET") + + async def stream_dispatch(_request): + return StreamingHttpResponse( + 200, + ((b"content-type", b"text/event-stream"),), + Source([unsafe + b"data: [DONE]\n\n"]), + ) + + checker = StaticChecker( + ContentInspectionPolicy(True, True, StreamBufferingPolicy(2, 0)), + output_decisions=[ContentBlocked("blocked")], + ) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client) + + assert response.status_code == 200 + assert b"INTERNAL-" not in response.content + assert b"SECRET" not in response.content + assert b'"code":"content_blocked"' in response.content + assert response.content.endswith(b"data: [DONE]\n\n") + + +@pytest.mark.asyncio +async def test_provider_error_event_is_preserved_without_output_check(): + stream = b'data: {"error":{"message":"upstream failed","type":"server_error"}}\n\ndata: [DONE]\n\n' + + async def stream_dispatch(_request): + return StreamingHttpResponse(200, ((b"content-type", b"text/event-stream"),), Source([stream])) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(2, 0))) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client) + + assert response.content == stream + assert [kind for kind, _ in checker.calls] == ["input"] + + +@pytest.mark.asyncio +async def test_streaming_provider_drift_is_hidden_in_native_error_events(): + hidden = b"unguarded reasoning" + event = stream_event(reasoning_content=hidden.decode()) + + async def stream_dispatch(_request): + return StreamingHttpResponse(200, ((b"content-type", b"text/event-stream"),), Source([event])) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client) + + assert response.status_code == 200 + assert hidden not in response.content + assert b'"code":"unsupported_chat_completions_response_shape"' in response.content + assert response.content.endswith(b"data: [DONE]\n\n") + + +@pytest.mark.asyncio +async def test_streaming_provider_http_error_is_preserved_and_bounded(): + provider_body = b'{"error":{"message":"rate limited","type":"provider_error"}}' + + async def stream_dispatch(_request): + return BufferedHttpResponse( + 429, + ((b"content-type", b"application/json"), (b"x-request-id", b"provider-error-id")), + provider_body, + ) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client) + + assert response.status_code == 429 + assert response.content == provider_body + assert response.headers["x-request-id"] == "provider-error-id" + assert [kind for kind, _ in checker.calls] == ["input"] + + async with make_client(checker, stream_dispatch, max_response_body_bytes=4) as client: + oversized = await post_stream(client) + + assert oversized.status_code == 502 + assert oversized.json()["error"]["code"] == "upstream_stream_failed" + assert provider_body not in oversized.content + + +@pytest.mark.asyncio +async def test_successful_non_sse_response_fails_before_headers_and_closes_source(): + source = Source([b'{"hidden":"secret-provider-payload"}']) + + async def stream_dispatch(_request): + return StreamingHttpResponse(200, ((b"content-type", b"application/json"),), source) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "unsupported_chat_completions_response_shape" + assert b"secret-provider-payload" not in response.content + assert source.closed is True + + +@pytest.mark.asyncio +async def test_stream_without_output_inspection_is_relayed_without_event_parsing(): + stream = b"opaque provider bytes" + source = Source([stream[:7], stream[7:]]) + + async def stream_dispatch(_request): + return StreamingHttpResponse(200, ((b"content-type", b"text/event-stream"),), source) + + checker = StaticChecker(ContentInspectionPolicy(True, False)) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client) + + assert response.content == stream + assert [kind for kind, _ in checker.calls] == ["input"] + assert source.closed is True + + +@pytest.mark.asyncio +async def test_upstream_iterator_failure_is_encoded_after_streaming_starts(): + source = Source([], RuntimeError("upstream disconnected")) + + async def stream_dispatch(_request): + return StreamingHttpResponse(200, ((b"content-type", b"text/event-stream"),), source) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch) as client: + response = await post_stream(client) + + assert response.status_code == 200 + assert b'"code":"upstream_stream_failed"' in response.content + assert response.content.endswith(b"data: [DONE]\n\n") + assert source.closed is True + + +def test_openai_endpoint_creates_fresh_stream_adapters(): + assert CHAT_COMPLETIONS_ENDPOINT.stream_adapter_factory is not None + + first = CHAT_COMPLETIONS_ENDPOINT.stream_adapter_factory() + second = CHAT_COMPLETIONS_ENDPOINT.stream_adapter_factory() + + assert first is not second + assert first.hooks is not second.hooks + assert first.classifier is second.classifier + + +def test_openapi_documents_both_chat_success_response_modes(): + async def dispatch(_request): + raise AssertionError + + app = FastAPI() + app.include_router( + create_openai_chat_router(checker=StaticChecker(ContentInspectionPolicy(True, True)), dispatch=dispatch) + ) + + success = app.openapi()["paths"]["/v1/chat/completions"]["post"]["responses"]["200"] + + assert success["description"] == "Provider-native successful response." + assert set(success["content"]) == {"application/json", "text/event-stream"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix", ["", "/proxy"]) +@pytest.mark.parametrize( + "method,path,status,code", + [ + ("GET", "/v1/chat/completions", 405, "method_not_allowed"), + ("HEAD", "/v1/chat/completions", 405, "method_not_allowed"), + ("POST", "/v1/chat/%2e/completions", 422, "non_canonical_path"), + ("POST", "/v1/chat/completions%3F", 422, "non_canonical_path"), + ("POST", "/v1/chat/completions%23", 422, "non_canonical_path"), + ("POST", "/v1%5Cchat/completions", 400, "invalid_request_path"), + ], +) +async def test_stream_requests_retain_native_route_rejections_before_dispatch(prefix, method, path, status, code): + async def dispatch(_request): + pytest.fail("a rejected streaming route must not dispatch") + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + app = FastAPI() + app.include_router( + create_openai_chat_router(checker=checker, dispatch=dispatch, stream_dispatch=dispatch), prefix=prefix + ) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://proxy.test") as client: + response = await client.request( + method, prefix + path, content=request_body(), headers={"content-type": "application/json"} + ) + + assert response.status_code == status + if method == "HEAD": + assert response.content == b"" + else: + assert response.json()["error"]["code"] == code + assert response.headers.get("allow") == ("POST" if status == 405 else None) + assert checker.calls == [] + assert status in {documented.status_code for documented in OPENAI_ERROR_MAPPING.responses} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "headers", [{"content-length": "999"}, {"content-length": "3", "transfer-encoding": "chunked"}] +) +async def test_stream_requests_use_shared_request_framing_validation(headers): + async def dispatch(_request): + pytest.fail("invalid request framing must stop before dispatch") + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch=dispatch) as client: + response = await post_stream(client, headers=headers) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_content_length" + assert checker.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [204, 205]) +async def test_bodyless_stream_success_fails_natively_before_headers(status): + source = Source([b"hidden provider content"]) + + async def dispatch(_request): + return StreamingHttpResponse( + status, ((b"content-type", b"text/event-stream"), (b"x-provider-secret", b"hidden")), source + ) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, stream_dispatch=dispatch) as client: + response = await post_stream(client) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "unsupported_chat_completions_response_shape" + assert "x-provider-secret" not in response.headers + assert source.closed is True diff --git a/tests/server/experimental/test_streaming_http.py b/tests/server/experimental/test_streaming_http.py index 4b1303e671..aee8f42c9e 100644 --- a/tests/server/experimental/test_streaming_http.py +++ b/tests/server/experimental/test_streaming_http.py @@ -24,7 +24,11 @@ from nemoguardrails.server.experimental import _stream_cleanup from nemoguardrails.server.experimental._buffered_kernel import OperationProjectionFailed from nemoguardrails.server.experimental._content_checker import ContentAllowed, StreamBufferingPolicy -from nemoguardrails.server.experimental._guarded_stream import StreamProcessingFailed, StreamUpstreamFailed +from nemoguardrails.server.experimental._guarded_stream import ( + StreamInspectionUnsupported, + StreamProcessingFailed, + StreamUpstreamFailed, +) from nemoguardrails.server.experimental._http_kernel import ( BufferedHttpRequest, BufferedHttpResponse, @@ -134,6 +138,8 @@ def render_outcome(outcome): code = "upstream_failed" elif isinstance(outcome, StreamProcessingFailed): code = "processing_failed" + elif isinstance(outcome, StreamInspectionUnsupported): + code = "inspection_unsupported" else: code = "stream_failure" return BufferedHttpResponse(502, ((b"content-type", b"text/plain"),), code.encode()) @@ -434,9 +440,10 @@ async def dispatch(_request): calls.append(True) raise AssertionError("unsafe policy must stop before dispatch") - with pytest.raises(ValueError, match="cannot release"): - await execute(dispatch, policy=StreamBufferingPolicy(1, 0, True)) + response = await execute(dispatch, policy=StreamBufferingPolicy(1, 0, True)) + assert response.status_code == 502 + assert await response_body(response) == b"inspection_unsupported" assert calls == [] From e43096deffb65fb0d137682a610637736ec24805 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:18:43 +0200 Subject: [PATCH 02/13] fix(server): document streaming only when it can be dispatched Advertise the text/event-stream success response only when the router has both a stream adapter and streaming dispatch, since streaming requests are rejected before dispatch otherwise. Remove the unused first construction of the buffered operation, which the factory replaced before returning. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/_guarded_proxy.py | 2 +- .../test_openai_chat_streaming_integration.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/nemoguardrails/server/experimental/_guarded_proxy.py b/nemoguardrails/server/experimental/_guarded_proxy.py index 547517baab..f347b24e90 100644 --- a/nemoguardrails/server/experimental/_guarded_proxy.py +++ b/nemoguardrails/server/experimental/_guarded_proxy.py @@ -207,7 +207,7 @@ async def handle_prepared_request( ) success_content = {"application/json": {"schema": {}}} - if endpoint.stream_adapter_factory is not None: + if endpoint.stream_adapter_factory is not None and stream_dispatch is not None: success_content["text/event-stream"] = {"schema": {"type": "string"}} documented_responses = errors.documented_responses documented_responses[200] = { diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py index 82367dc594..0fef1f1f92 100644 --- a/tests/server/experimental/test_openai_chat_streaming_integration.py +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -408,19 +408,27 @@ def test_openai_endpoint_creates_fresh_stream_adapters(): assert first.classifier is second.classifier -def test_openapi_documents_both_chat_success_response_modes(): +@pytest.mark.parametrize( + ("configure_stream_dispatch", "media_types"), + [(True, {"application/json", "text/event-stream"}), (False, {"application/json"})], +) +def test_openapi_documents_chat_success_modes_that_can_be_dispatched(configure_stream_dispatch, media_types): async def dispatch(_request): raise AssertionError app = FastAPI() app.include_router( - create_openai_chat_router(checker=StaticChecker(ContentInspectionPolicy(True, True)), dispatch=dispatch) + create_openai_chat_router( + checker=StaticChecker(ContentInspectionPolicy(True, True)), + dispatch=dispatch, + stream_dispatch=dispatch if configure_stream_dispatch else None, + ) ) success = app.openapi()["paths"]["/v1/chat/completions"]["post"]["responses"]["200"] assert success["description"] == "Provider-native successful response." - assert set(success["content"]) == {"application/json", "text/event-stream"} + assert set(success["content"]) == media_types @pytest.mark.asyncio From 4df23015ca8a8af3f3d41336f2e15e83e160e49e Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:49:53 +0200 Subject: [PATCH 03/13] docs(server): describe the Chat streaming route The endpoint now binds the stream classifier and a fresh hook instance per request, and the router dispatches either response mode. State that the buffered contract export stays buffered-only. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../providers/openai/chat_completions/README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md index cebb42ba79..89c0daf4aa 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md @@ -99,8 +99,10 @@ errors as terminal outcomes, and frames native errors. These checks require history and do not belong in field annotations or a stateless classifier. The projections and classifier can be used independently of endpoint wiring. -This stage does not bind them to the HTTP endpoint or extend the shared buffered -contract exporter with streaming declarations. +The endpoint binds the classifier and a fresh hook instance per request through +the stream adapter factory. The router selects buffered or streaming dispatch +after the shared request projection and input check. Streaming requests require +an injected streaming dispatcher; without one, they fail before dispatch. ## Buffered contract export @@ -141,10 +143,12 @@ The [exported buffered contract](../../../contracts/openai/_generated/chat-compl gets its field policy from the Python models and endpoint labels, route, and error codes from `CHAT_COMPLETIONS_ENDPOINT`. Do not edit the artifact by hand. This is a buffered-only export: there is no stream section or streaming hook. -The request model recognizes the `stream` flag, but this integration still -rejects streaming requests before dispatch. Replacement eligibility is declared -by the models, while applying replacement outcomes remains unsupported by this -integration. Exporting the policy does not enable either runtime feature. +The request model recognizes the `stream` flag, and the router supports both +response modes when their dispatchers are supplied. The export remains +buffered-only even when the endpoint binds a stream adapter. Streaming export +is separate work. Replacement eligibility is declared by the models, while +applying replacement outcomes remains unsupported by this integration. +Exporting the policy does not enable runtime features. A successful response whose assistant content is empty or null is relayed unchanged without output checks, but only after the full response passes the From c53d57f0480d71a49277a5690bd5afa5eae21d74 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:34:20 +0200 Subject: [PATCH 04/13] fix(server): honor endpoint codes for streaming failures Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/_guarded_proxy.py | 22 +++++++++++-- .../test_openai_chat_streaming_integration.py | 32 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/nemoguardrails/server/experimental/_guarded_proxy.py b/nemoguardrails/server/experimental/_guarded_proxy.py index f347b24e90..8023244354 100644 --- a/nemoguardrails/server/experimental/_guarded_proxy.py +++ b/nemoguardrails/server/experimental/_guarded_proxy.py @@ -15,7 +15,7 @@ """Compose provider declarations with the guarded HTTP pipeline.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace from pydantic import ValidationError @@ -33,6 +33,7 @@ ) from nemoguardrails.server.experimental._guarded_stream import ( StreamInspectionUnsupported, + StreamOutcome, UnsupportedStreamInspection, ) from nemoguardrails.server.experimental._http_kernel import ( @@ -171,6 +172,21 @@ def project_response( else request_model.model_json_schema() ) + def render_stream_outcome(outcome: StreamOutcome) -> BufferedHttpResponse: + """Bind generic stream shape failures to the endpoint's declared codes.""" + if ( + isinstance(outcome, OperationProjectionFailed) + and type(outcome.failure) is UnsupportedGuardedPayload + and outcome.failure.code is None + ): + code = ( + endpoint.unsupported_request_code + if outcome.stage is InspectionStage.INPUT + else endpoint.unsupported_response_code + ) + outcome = replace(outcome, failure=UnsupportedGuardedPayload(str(outcome.failure), code)) + return errors.renderer(outcome) + async def handle_prepared_request( prepared_input: PreparedOperationInput[GuardableProviderRequest], ): @@ -178,7 +194,7 @@ async def handle_prepared_request( if not guardable.streaming: return None if stream_dispatch is None: - return errors.renderer( + return render_stream_outcome( OperationProjectionFailed( InspectionStage.INPUT, UnsupportedGuardedPayload("The guarded endpoint does not have streaming dispatch configured."), @@ -201,7 +217,7 @@ async def handle_prepared_request( streaming_policy=streaming_policy, input_message=guardable.target.message, adapter=adapter_factory(), - render_outcome=errors.renderer, + render_outcome=render_stream_outcome, max_event_bytes=max_stream_event_bytes, max_pending_bytes=max_pending_stream_bytes, ) diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py index 0fef1f1f92..d9471b86b9 100644 --- a/tests/server/experimental/test_openai_chat_streaming_integration.py +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -14,6 +14,7 @@ # limitations under the License. import json +from dataclasses import replace import httpx import pytest @@ -114,6 +115,37 @@ async def reject_buffered(_request): return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://proxy.test") +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["missing_dispatch", "event", "media_type", "encoding"]) +async def test_stream_errors_honor_endpoint_codes(failure, monkeypatch): + """Endpoint codes survive failures before and after streaming headers.""" + from nemoguardrails.server.experimental.providers.openai import integration + + endpoint = replace( + CHAT_COMPLETIONS_ENDPOINT, + unsupported_request_code="example_request_shape", + unsupported_response_code="example_response_shape", + ) + monkeypatch.setattr(integration, "CHAT_COMPLETIONS_ENDPOINT", endpoint) + source = Source([b'data: {"unexpected": "private-stream-content"}\n\n']) + headers = [(b"content-type", b"text/plain" if failure == "media_type" else b"text/event-stream")] + if failure == "encoding": + headers.append((b"content-encoding", b"gzip")) + + async def dispatch(_request): + return StreamingHttpResponse(200, tuple(headers), source) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, None if failure == "missing_dispatch" else dispatch) as client: + response = await post_stream(client) + expected = "example_request_shape" if failure == "missing_dispatch" else "example_response_shape" + assert ('"code":"' + expected + '"').encode() in response.content + assert b"private-stream-content" not in response.content + assert response.status_code == (422 if failure == "missing_dispatch" else 200 if failure == "event" else 502) + if failure != "missing_dispatch": + assert source.closed is True + + async def post_stream(client, body=None, **kwargs): headers = {"content-type": "application/json", **kwargs.pop("headers", {})} return await client.post("/v1/chat/completions", content=body or request_body(), headers=headers, **kwargs) From eff5b949f56506c63759c28324bc7d9ee6215cf5 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:43:58 +0200 Subject: [PATCH 05/13] feat(server): export bound streaming guard contracts Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/_guarded_proxy.py | 10 +- .../contracts/guard-contract.schema.json | 6 + .../experimental/contracts/openai/README.md | 22 +- ...guard.yaml => chat-completions.guard.yaml} | 219 ++++++++++++++++ .../experimental/provider/contract_export.py | 99 ++++++- .../server/experimental/provider/endpoint.py | 14 +- .../server/experimental/provider/stream.py | 39 ++- .../openai/chat_completions/README.md | 41 +-- .../openai/chat_completions/endpoint.py | 4 +- .../experimental/test_openai_chat_contract.py | 7 +- .../test_openai_chat_contract_export.py | 8 +- .../test_openai_chat_streaming_integration.py | 6 +- .../test_openai_chat_streaming_projections.py | 8 +- .../test_stream_contract_export.py | 245 ++++++++++++++++++ 14 files changed, 659 insertions(+), 69 deletions(-) rename nemoguardrails/server/experimental/contracts/openai/_generated/{chat-completions.buffered.guard.yaml => chat-completions.guard.yaml} (61%) create mode 100644 tests/server/experimental/test_stream_contract_export.py diff --git a/nemoguardrails/server/experimental/_guarded_proxy.py b/nemoguardrails/server/experimental/_guarded_proxy.py index 8023244354..2391c04a73 100644 --- a/nemoguardrails/server/experimental/_guarded_proxy.py +++ b/nemoguardrails/server/experimental/_guarded_proxy.py @@ -102,7 +102,7 @@ def _prepare_guardable_request( guarded_schema_error(error, "request"), endpoint.unsupported_request_code ) from error streaming = projection.streams_response - if streaming and endpoint.stream_adapter_factory is None: + if streaming and endpoint.stream is None: raise UnsupportedGuardedPayload( "The guarded endpoint does not support streaming responses.", endpoint.unsupported_request_code ) @@ -207,8 +207,8 @@ async def handle_prepared_request( UnsupportedStreamInspection("Guarded output streaming requires a buffering policy.") ) ) - adapter_factory = endpoint.stream_adapter_factory - if adapter_factory is None: + stream_binding = endpoint.stream + if stream_binding is None: raise AssertionError("A streaming request requires an endpoint stream adapter.") return await execute_streaming_http( guardable.request, @@ -216,14 +216,14 @@ async def handle_prepared_request( checker=prepared_input.resolved.checker, streaming_policy=streaming_policy, input_message=guardable.target.message, - adapter=adapter_factory(), + adapter=stream_binding.create_adapter(), render_outcome=render_stream_outcome, max_event_bytes=max_stream_event_bytes, max_pending_bytes=max_pending_stream_bytes, ) success_content = {"application/json": {"schema": {}}} - if endpoint.stream_adapter_factory is not None and stream_dispatch is not None: + if endpoint.stream is not None and stream_dispatch is not None: success_content["text/event-stream"] = {"schema": {"type": "string"}} documented_responses = errors.documented_responses documented_responses[200] = { diff --git a/nemoguardrails/server/experimental/contracts/guard-contract.schema.json b/nemoguardrails/server/experimental/contracts/guard-contract.schema.json index f4fa9c67b5..cf4921857a 100644 --- a/nemoguardrails/server/experimental/contracts/guard-contract.schema.json +++ b/nemoguardrails/server/experimental/contracts/guard-contract.schema.json @@ -456,6 +456,12 @@ "pattern": { "type": "string" }, + "minimum": { + "type": "number" + }, + "maximum": { + "type": "number" + }, "properties": { "type": "object", "additionalProperties": { diff --git a/nemoguardrails/server/experimental/contracts/openai/README.md b/nemoguardrails/server/experimental/contracts/openai/README.md index 1f47cb7f3c..6b1506b54b 100644 --- a/nemoguardrails/server/experimental/contracts/openai/README.md +++ b/nemoguardrails/server/experimental/contracts/openai/README.md @@ -11,6 +11,7 @@ behavior. See [Guard contracts](../README.md) for the document vocabulary. | --- | --- | | Request | One user message with non-empty string content at `messages[0].content`. | | Buffered response | One assistant choice with string content at `choices[0].message.content`. Empty or null content means the response has nothing to inspect, because every other reviewed field is content-free. The projection reports this through `has_text`, and the buffered integration relays such a response without output checks. | +| Streamed response | At most one choice, with integer index zero and optional assistant text at `choices[0].delta.content`. Missing or null text is metadata; an empty choices array allows usage chunks. Reviewed root metadata passes through. | | Constrained values | User/assistant roles, single-item arrays, and request `n` constrained to one. | | Unsupported content | Tool, audio, multimodal, refusal, participant name, and separate reasoning content where explicitly disabled. | | Opaque data | Reviewed provider-owned metadata and controls, not additional guarded subjects. | @@ -26,7 +27,7 @@ request `logprobs` accepts null or `false`. Log probabilities are unsupported: they carry token text that rails do not inspect. Request `logprobs` accepts only `false` or null, request `top_logprobs` -accepts only null, and buffered response `choices[0].logprobs` accepts only null. +accepts only null, and both buffered and stream `choices[0].logprobs` accept only null. The request check applies even when output inspection is off. Text replacement eligibility is separate from endpoint support for replacement @@ -35,11 +36,20 @@ not inspected. The policy still declares that non-empty annotations would block text replacement. Unrelated provider data must remain intact. The Python projections, bindings, and endpoint determine exact acceptance and -runtime behavior. Their machine-readable contract belongs with the integration, -not in a separately maintained handwritten policy here. Recognizing the request -`stream` flag does not itself provide a streaming endpoint. Streaming needs its -own event classification, lifecycle handling, and implementation tests; the -buffered boundary is not a claim about accepted stream events. +runtime behavior. The [exported contract](_generated/chat-completions.guard.yaml) +describes buffered payloads and the bound stream classifier, not a separately +maintained handwritten policy. The stream root, choice, delta, and error envelope +are closed. Error details are configurable, but the classifier validates with +the closed default, so unreviewed error fields are rejected too. + +The stream export includes event selection, roles, missing-text handling, and +SSE sentinel/non-data declarations. Stateful hooks enforce completion and event +ordering and frame provider-native errors; they remain handwritten and are not +serialized. Exporting a contract does not enable streaming dispatch or reproduce +arbitrary Python validation. For example, the runtime rejects choice index +`0.0`, while JSON Schema treats it as the integer zero. See the integration's +[export guide](../../providers/openai/chat_completions/README.md#contract-export) +for generation, drift checks, and limitations. ## Provider provenance diff --git a/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.buffered.guard.yaml b/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml similarity index 61% rename from nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.buffered.guard.yaml rename to nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml index 66583de67f..44031228ee 100644 --- a/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.buffered.guard.yaml +++ b/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml @@ -390,3 +390,222 @@ integration: operation_label: OpenAI Chat Completions unsupported_request_code: unsupported_chat_completions_shape unsupported_response_code: unsupported_chat_completions_response_shape +stream: + title: openai.chat_completions.stream.text.v1 + oneOf: + - additionalProperties: false + description: Allow one text choice or an empty usage chunk, preserving root metadata. + properties: + choices: + items: + additionalProperties: false + description: Require integer choice zero and reject unreviewed choice-level + content. + properties: + delta: + additionalProperties: false + description: Accept assistant text or metadata-only deltas within a + closed boundary. + properties: + audio: + default: null + type: 'null' + x-nemo-guardrails: + classification: constrained + extension: true + gate: disabled + reason: core_capability.audio_content + content: + default: null + x-nemo-guardrails: + classification: guarded + subject: + kind: text + role: assistant + oneOf: + - type: string + - type: 'null' + function_call: + default: null + type: 'null' + x-nemo-guardrails: + classification: constrained + gate: disabled + reason: core_capability.tool_content + reasoning_content: + default: null + type: 'null' + x-nemo-guardrails: + classification: constrained + extension: true + gate: disabled + reason: core_capability.reasoning_content + refusal: + default: null + type: 'null' + x-nemo-guardrails: + classification: constrained + gate: disabled + reason: core_capability.refusal_content + role: + default: null + x-nemo-guardrails: + classification: constrained + oneOf: + - const: assistant + type: string + - type: 'null' + tool_calls: + default: null + x-nemo-guardrails: + classification: constrained + reason: core_capability.tool_content + oneOf: + - items: {} + maxItems: 0 + type: array + - type: 'null' + title: ChatCompletionsStreamDeltaProjection + type: object + x-nemo-guardrails: + classification: guarded + source: '#/components/schemas/ChatCompletionStreamResponseDelta' + unknown_fields: forbid + reject_case_aliases: true + finish_reason: + default: null + x-nemo-guardrails: + classification: opaque + index: + maximum: 0 + minimum: 0 + type: integer + x-nemo-guardrails: + classification: constrained + logprobs: + default: null + type: 'null' + x-nemo-guardrails: + classification: constrained + gate: disabled + reason: provider_integrity.token_logprobs + required: + - delta + - index + title: ChatCompletionsStreamChoiceProjection + type: object + x-nemo-guardrails: + unknown_fields: forbid + reject_case_aliases: true + maxItems: 1 + type: array + x-nemo-guardrails: + classification: guarded + created: + x-nemo-guardrails: + classification: opaque + id: + x-nemo-guardrails: + classification: opaque + model: + x-nemo-guardrails: + classification: opaque + moderation: + x-nemo-guardrails: + classification: opaque + obfuscation: + x-nemo-guardrails: + classification: opaque + object: + const: chat.completion.chunk + type: string + x-nemo-guardrails: + classification: constrained + service_tier: + x-nemo-guardrails: + classification: opaque + system_fingerprint: + x-nemo-guardrails: + classification: opaque + usage: + x-nemo-guardrails: + classification: opaque + required: + - choices + - object + title: ChatCompletionsStreamPayloadProjection + type: object + x-nemo-guardrails: + source: '#/components/schemas/CreateChatCompletionStreamResponse' + unknown_fields: forbid + reject_case_aliases: true + model: ChatCompletionsStreamPayloadProjection + event: + classification: guarded_delta + variants: + - source_schema: CreateChatCompletionStreamResponse + shape: chat.completion.chunk:content + match: + object: chat.completion.chunk + missing_text: opaque + missing_text_shape: chat.completion.chunk:metadata + - additionalProperties: false + description: Accept only an error envelope, never an error mixed with generated + content. + properties: + error: + additionalProperties: false + description: Validate the native error message without treating it as guarded + text. + properties: + code: + default: null + x-nemo-guardrails: + classification: opaque + message: + type: string + x-nemo-guardrails: + classification: constrained + misalignment: + default: null + x-nemo-guardrails: + classification: opaque + param: + default: null + x-nemo-guardrails: + classification: opaque + type: + default: null + x-nemo-guardrails: + classification: opaque + required: + - message + title: ChatCompletionsStreamErrorDetailsProjection + type: object + x-nemo-guardrails: + classification: constrained + source: '#/components/schemas/Error' + unknown_fields: configurable + reject_case_aliases: true + required: + - error + title: ChatCompletionsStreamErrorProjection + type: object + x-nemo-guardrails: + source: '#/components/schemas/ErrorResponse' + unknown_fields: forbid + reject_case_aliases: true + model: ChatCompletionsStreamErrorProjection + event: + classification: provider_error + variants: + - source_schema: ErrorResponse + shape: error + required_fields: + - error + x-nemo-guardrails: + transport: + require_sse_event: false + non_data_shape: keepalive + sentinels: + '[DONE]': '[DONE]' diff --git a/nemoguardrails/server/experimental/provider/contract_export.py b/nemoguardrails/server/experimental/provider/contract_export.py index 3655227cfe..bd1d13ffa6 100644 --- a/nemoguardrails/server/experimental/provider/contract_export.py +++ b/nemoguardrails/server/experimental/provider/contract_export.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Export buffered guard contracts from trusted Python endpoint declarations. +"""Export guard contracts from trusted Python endpoint declarations. Provider integrations supply an endpoint and document identity, not exporter implementations. Model annotations describe payload policy; endpoint metadata @@ -50,6 +50,84 @@ payload_contract, text_location, ) +from nemoguardrails.server.experimental.provider.stream import StreamBinding, StreamCapabilityProfile, StreamEventRole +from nemoguardrails.server.experimental.provider.stream_classifier import ( + StreamClassifier, + StreamClassifierDefinition, + StreamEventRule, +) +from nemoguardrails.server.experimental.provider.stream_policy import build_policy_stream_classifier + + +def _export_stream_event(rule: StreamEventRule) -> dict[str, Any]: + """Describe a typed event rule without dropping unsupported selection or fallback semantics.""" + if type(rule) is not StreamEventRule: + raise TypeError("Stream contract export requires standard declarative event rules") + if not issubclass(rule.model, PolicyModel) or not rule.model.policy.source: + raise ValueError("Stream event export requires an explicit policy source schema") + if len(dict(rule.match)) != len(rule.match) or any(type(value) not in (str, int, bool) for _, value in rule.match): + raise ValueError("Stream event export requires unique match keys with string, integer, or boolean values") + roles = { + StreamEventRole.GUARDED_TEXT: "guarded_delta", + StreamEventRole.TEXT_SNAPSHOT: "snapshot", + StreamEventRole.OPAQUE_METADATA: "opaque", + StreamEventRole.PROVIDER_ERROR: "provider_error", + } + variant: dict[str, Any] = {"source_schema": rule.model.policy.source, "shape": rule.shape} + if rule.match: + variant["match"] = dict(rule.match) + if rule.required_fields: + variant["required_fields"] = sorted(rule.required_fields) + event: dict[str, Any] = {"classification": roles[rule.role], "variants": [variant]} + if rule.role in {StreamEventRole.GUARDED_TEXT, StreamEventRole.TEXT_SNAPSHOT}: + if rule.missing_text_role is None: + event["missing_text"] = "reject" + elif rule.missing_text_role is StreamEventRole.OPAQUE_METADATA and rule.missing_text_value is None: + event["missing_text"] = "opaque" + elif rule.missing_text_role is rule.role and rule.missing_text_value == "": + event["missing_text"] = "empty" + else: + raise ValueError("Stream missing-text fallback cannot be represented by the contract") + if rule.missing_text_shape is not None: + event["missing_text_shape"] = rule.missing_text_shape + schema = export_payload_schema(rule.model, projection_id=rule.model.__name__) + schema[EXTENSION]["event"] = event + return schema + + +def _export_stream(binding: StreamBinding) -> dict[str, Any]: + """Export supported classifier declarations, never instantiate or serialize protocol hooks.""" + classifier = binding.classifier + if type(classifier) is not StreamClassifier: + raise TypeError("Stream contract export requires the standard declarative StreamClassifier") + if classifier.contract.profile is not StreamCapabilityProfile.SINGLE_TEXT_DELTA_V1: + raise ValueError("Stream capability profile cannot be represented by the contract") + build_policy_stream_classifier( + StreamClassifierDefinition( + subject=classifier.subject, + contract=classifier.contract, + rules=classifier.rules, + sentinels=classifier.sentinels, + non_data_shape=classifier.non_data_shape, + event_type_field=classifier.event_type_field, + require_event_type=classifier.require_event_type, + ) + ) + transport: dict[str, Any] = {"require_sse_event": classifier.require_event_type} + if classifier.event_type_field is not None: + transport["data_discriminator"] = classifier.event_type_field + if classifier.non_data_shape is not None: + transport["non_data_shape"] = classifier.non_data_shape + if classifier.sentinels: + try: + transport["sentinels"] = {value.decode("utf-8"): shape for value, shape in classifier.sentinels} + except UnicodeDecodeError as error: + raise ValueError("Stream contract sentinels must be UTF-8 text") from error + return { + "title": classifier.contract.projection_id, + "oneOf": [_export_stream_event(rule) for rule in classifier.rules], + EXTENSION: {"transport": transport}, + } def _validate_policy_binding( @@ -86,17 +164,17 @@ def _validate_stream_selector(model: type[GuardedRequestModel]) -> None: def export_guard_contract(endpoint: GuardedJsonEndpoint) -> dict[str, Any]: - """Describe the buffered policy of the models actually bound to an endpoint. + """Describe the payload and stream policy actually bound to an endpoint. Args: endpoint: A trusted endpoint with policy-annotated request and response models and their runtime bindings. Returns: - A fresh document with payload schemas, profile, and endpoint labels. - The optional request stream selector is descriptive; there is no stream - event section. Header/query revision bindings, alternate route ownership, - and arbitrary Python behavior are not serialized. + A fresh document with payload schemas, profile, endpoint labels, and + optional stream event rules and transport metadata. Stateful hooks, + header/query revision bindings, alternate route ownership, and arbitrary + Python behavior are not serialized. Raises: TypeError: The endpoint or its models do not support policy export. @@ -130,7 +208,7 @@ def export_guard_contract(endpoint: GuardedJsonEndpoint) -> dict[str, Any]: "unsupported_request_code": endpoint.unsupported_request_code, "unsupported_response_code": endpoint.unsupported_response_code, } - return { + document = { "version": CONTRACT_VERSION, "operationId": endpoint.provider_operation_id, "profile": request_contract.profile.value, @@ -138,6 +216,9 @@ def export_guard_contract(endpoint: GuardedJsonEndpoint) -> dict[str, Any]: "response": response, "integration": integration, } + if endpoint.stream is not None: + document["stream"] = _export_stream(endpoint.stream) + return document def _load_endpoint(reference: str) -> GuardedJsonEndpoint: @@ -155,14 +236,14 @@ def _load_endpoint(reference: str) -> GuardedJsonEndpoint: def main() -> None: - """Validate and emit a buffered contract, or check an artifact for exact drift. + """Validate and emit a guard contract, or check an artifact for exact drift. The endpoint argument imports trusted Python code. With --output, create parent directories and overwrite the selected file. With --check, leave files untouched and exit with status 1 on missing or differing content. With neither option, write YAML to stdout. Invalid declarations fail before file writes. """ - parser = argparse.ArgumentParser(description="Export a buffered guard contract from a trusted Python endpoint.") + parser = argparse.ArgumentParser(description="Export a guard contract from a trusted Python endpoint.") parser.add_argument("endpoint", type=_load_endpoint, help="trusted Python module:attribute (imports code)") destination = parser.add_mutually_exclusive_group() destination.add_argument("--output", type=Path) diff --git a/nemoguardrails/server/experimental/provider/endpoint.py b/nemoguardrails/server/experimental/provider/endpoint.py index 845ea747e9..047258c96e 100644 --- a/nemoguardrails/server/experimental/provider/endpoint.py +++ b/nemoguardrails/server/experimental/provider/endpoint.py @@ -16,7 +16,6 @@ """Declare provider endpoints handled by the guarded JSON pipeline.""" import re -from collections.abc import Callable from dataclasses import dataclass from nemoguardrails.server.experimental._http_paths import GuardedOperationPath @@ -25,7 +24,7 @@ GuardedRequestModel, validate_payload_projection_contract, ) -from nemoguardrails.server.experimental.provider.stream import ProviderStreamAdapter, StreamProjectionContract +from nemoguardrails.server.experimental.provider.stream import StreamBinding from nemoguardrails.server.experimental.provider.transport import ProviderApiRevisionBinding @@ -44,7 +43,7 @@ class GuardedJsonEndpoint: contract_name: str | None = None method: str = "POST" api_revision: ProviderApiRevisionBinding | None = None - stream_adapter_factory: Callable[[], ProviderStreamAdapter] | None = None + stream: StreamBinding | None = None operation_paths: tuple[GuardedOperationPath, ...] = () def __post_init__(self) -> None: @@ -78,10 +77,5 @@ def __post_init__(self) -> None: response_contract = validate_payload_projection_contract(self.guarded_response_model, "response") if request_contract.profile != response_contract.profile: raise ValueError("Guarded request and response capability profiles must match.") - if self.stream_adapter_factory is not None: - adapter = self.stream_adapter_factory() - required_methods = ("classify_event", "validate_end_of_stream", "encode_error") - if any(not callable(getattr(adapter, method, None)) for method in required_methods): - raise ValueError("The stream adapter factory must create a provider stream adapter.") - if not isinstance(getattr(adapter, "contract", None), StreamProjectionContract): - raise ValueError("The stream adapter must declare its stream projection contract.") + if self.stream is not None and not isinstance(self.stream, StreamBinding): + raise TypeError("The endpoint stream must be a StreamBinding.") diff --git a/nemoguardrails/server/experimental/provider/stream.py b/nemoguardrails/server/experimental/provider/stream.py index a175d6d604..266bc2266c 100644 --- a/nemoguardrails/server/experimental/provider/stream.py +++ b/nemoguardrails/server/experimental/provider/stream.py @@ -181,7 +181,7 @@ def encode_error(self, rendered_body: bytes) -> Sequence[bytes]: @dataclass(slots=True) class ClassifiedStreamAdapter: - """Compose a generated classifier with handwritten protocol hooks.""" + """Compose a stateless classifier with handwritten protocol hooks.""" classifier: ProviderStreamClassifier hooks: ProviderStreamHooks @@ -210,10 +210,35 @@ def encode_error(self, rendered_body: bytes) -> Sequence[bytes]: return self.hooks.encode_error(rendered_body) -def create_classified_stream_adapter_factory( - classifier: ProviderStreamClassifier, - hooks_factory: Callable[[], ProviderStreamHooks], -) -> Callable[[], ProviderStreamAdapter]: - """Create a zero-argument adapter factory for an endpoint declaration.""" +@dataclass(frozen=True, slots=True) +class StreamBinding: + """Bind shared classification to a fresh set of protocol hooks per request. + + Construction checks declarations without running the hook factory. Runtime + uses create_adapter; inspection can read the classifier directly. Factories + must return independent hook instances so lifecycle state is never shared. + Handwritten classifiers are supported at runtime, but are not automatically + representable by a guard contract. + """ - return lambda: ClassifiedStreamAdapter(classifier, hooks_factory()) + classifier: ProviderStreamClassifier + hooks_factory: Callable[[], ProviderStreamHooks] + + def __post_init__(self) -> None: + """Reject malformed bindings without creating request state.""" + if not isinstance(getattr(self.classifier, "contract", None), StreamProjectionContract): + raise TypeError("A stream binding requires a classifier with a stream projection contract.") + if not callable(getattr(self.classifier, "classify_event", None)): + raise TypeError("A stream binding requires a callable classifier.") + if not callable(self.hooks_factory): + raise TypeError("A stream binding requires a callable hook factory.") + + def create_adapter(self) -> ClassifiedStreamAdapter: + """Create request-local hooks and validate their protocol interface.""" + hooks = self.hooks_factory() + if any( + not callable(getattr(hooks, method, None)) + for method in ("observe_event", "validate_end_of_stream", "encode_error") + ): + raise TypeError("The hook factory must create provider stream hooks.") + return ClassifiedStreamAdapter(self.classifier, hooks) diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md index 89c0daf4aa..88756d99f5 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md @@ -99,19 +99,23 @@ errors as terminal outcomes, and frames native errors. These checks require history and do not belong in field annotations or a stateless classifier. The projections and classifier can be used independently of endpoint wiring. -The endpoint binds the classifier and a fresh hook instance per request through -the stream adapter factory. The router selects buffered or streaming dispatch +The endpoint owns one `StreamBinding`: the classifier and a hook factory. +Runtime creates fresh hooks per request; endpoint construction and contract +export never invoke the factory. The router selects buffered or streaming dispatch after the shared request projection and input check. Streaming requests require an injected streaming dispatcher; without one, they fail before dispatch. -## Buffered contract export +## Contract export The shared [contract exporter](../../../provider/contract_export.py) takes the runtime endpoint, including its declared document identity. No per-operation exporter is needed. Before export, it checks coverage, text bindings, replacement restrictions, and any -declared stream selector against the field policy. Custom extraction and mismatched -bindings are rejected rather than described as if they were equivalent. +declared stream selector against the field policy. Stream rules are checked against +typed text paths and exact shape coverage. Custom extraction, mismatched bindings, +and unsupported event selectors or fallbacks are rejected rather than silently +omitted. A handwritten classifier can still run through `StreamBinding`, but the +exporter requires the standard declarative classifier. ```python from nemoguardrails.server.experimental.provider.contract_export import export_guard_contract @@ -125,7 +129,7 @@ Run the shared CLI from the repository root: ```bash uv run --locked python -m nemoguardrails.server.experimental.provider.contract_export \ nemoguardrails.server.experimental.providers.openai.chat_completions.endpoint:CHAT_COMPLETIONS_ENDPOINT \ - --output nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.buffered.guard.yaml + --output nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml ``` Use `--check` instead of `--output` with the same path to detect drift. Without @@ -139,15 +143,17 @@ requests or untrusted documents. The CLI does not scan providers or load runtime configuration from YAML. Its YAML/JSON Schema dependencies are needed only for serialization and format validation, not endpoint construction. -The [exported buffered contract](../../../contracts/openai/_generated/chat-completions.buffered.guard.yaml) +The [exported contract](../../../contracts/openai/_generated/chat-completions.guard.yaml) gets its field policy from the Python models and endpoint labels, route, and error codes from `CHAT_COMPLETIONS_ENDPOINT`. Do not edit the artifact by hand. -This is a buffered-only export: there is no stream section or streaming hook. -The request model recognizes the `stream` flag, and the router supports both -response modes when their dispatchers are supplied. The export remains -buffered-only even when the endpoint binds a stream adapter. Streaming export -is separate work. Replacement eligibility is declared by the models, while -applying replacement outcomes remains unsupported by this integration. +Its stream section describes the same classifier bound at runtime: event payload +policy, roles, predicates, missing-text handling, sentinel, and non-data behavior. +An endpoint without a stream binding exports only its buffered boundary. +Handwritten hooks are not serialized: ordering, completion checks, and native +error framing remain in Python. The request model recognizes the `stream` flag, +and the router supports both response modes when their dispatchers are supplied. +Replacement eligibility is declared by the models, while applying replacement +outcomes remains unsupported by this integration. Exporting the policy does not enable runtime features. A successful response whose assistant content is empty or null is relayed @@ -167,9 +173,12 @@ response cannot be inspected. A successful response that is still encoded is rejected, not relayed. Without output checks, the client's own preference is forwarded and the response is relayed unchanged. -The shared exporter describes buffered payload policy and endpoint labels, not -arbitrary transport behavior: header/query API-revision bindings and alternate -route ownership are not serialized. +The shared exporter describes payload policy, declared stream semantics, and +endpoint labels, not arbitrary Python validation or transport behavior: +header/query API-revision bindings and alternate route ownership are not +serialized. JSON Schema also cannot distinguish the integer `0` from `0.0`; +the stream's Python validator requires an actual integer choice index. Export is +an inspection document, not proof of complete runtime value equivalence. Nullable annotations are exported as disjoint array/null `oneOf` branches; the array branch has `maxItems: 0` because citation content is not inspected. diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py b/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py index 5ca35e8d73..dbf8e71986 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/endpoint.py @@ -17,7 +17,7 @@ from nemoguardrails.server.experimental._http_paths import GuardedOperationPath from nemoguardrails.server.experimental.provider.endpoint import GuardedJsonEndpoint -from nemoguardrails.server.experimental.provider.stream import create_classified_stream_adapter_factory +from nemoguardrails.server.experimental.provider.stream import StreamBinding from nemoguardrails.server.experimental.providers.openai.chat_completions.request_binding import ( ChatCompletionsGuardedRequest, ) @@ -48,7 +48,7 @@ frozenset({"POST"}), ), ), - stream_adapter_factory=create_classified_stream_adapter_factory( + stream=StreamBinding( STREAM_CLASSIFIER, ChatCompletionsStreamHooks, ), diff --git a/tests/server/experimental/test_openai_chat_contract.py b/tests/server/experimental/test_openai_chat_contract.py index c392bbf1a6..d4539ef7d6 100644 --- a/tests/server/experimental/test_openai_chat_contract.py +++ b/tests/server/experimental/test_openai_chat_contract.py @@ -132,9 +132,10 @@ def test_chat_exports_conform_to_the_guard_contract_schema(): assert _contract_errors(ChatCompletionsGuardedRequestProjection, ChatCompletionsGuardedResponseProjection) == [] -def test_contract_schema_check_catches_unexportable_constraints(): +@pytest.mark.parametrize(("bound", "exportable"), [({"ge": 0}, True), ({"le": 0}, True), ({"gt": 0}, False)]) +def test_contract_schema_supports_only_inclusive_numeric_bounds(bound, exportable): class Response(PolicyModel): text: Annotated[str, guarded("assistant")] - count: Annotated[int, Field(ge=0), constrained()] + count: Annotated[int, Field(**bound), constrained()] - assert _contract_errors(ChatCompletionsGuardedRequestProjection, Response) + assert (_contract_errors(ChatCompletionsGuardedRequestProjection, Response) == []) is exportable diff --git a/tests/server/experimental/test_openai_chat_contract_export.py b/tests/server/experimental/test_openai_chat_contract_export.py index ef440e73f3..2f46e066bf 100644 --- a/tests/server/experimental/test_openai_chat_contract_export.py +++ b/tests/server/experimental/test_openai_chat_contract_export.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Check that the buffered contract stays derived from the runtime endpoint.""" +"""Check that the operation contract stays derived from the runtime endpoint.""" import json import subprocess @@ -32,19 +32,19 @@ ROOT = Path(__file__).parents[3] CONTRACTS = ROOT / "nemoguardrails/server/experimental/contracts" -EXPORTED = CONTRACTS / "openai/_generated/chat-completions.buffered.guard.yaml" +EXPORTED = CONTRACTS / "openai/_generated/chat-completions.guard.yaml" MODULE = "nemoguardrails.server.experimental.provider.contract_export" ENDPOINT = "nemoguardrails.server.experimental.providers.openai.chat_completions.endpoint:CHAT_COMPLETIONS_ENDPOINT" CLI = [sys.executable, "-m", MODULE, ENDPOINT] -def test_buffered_export_matches_checked_in_artifact_and_format(): +def test_operation_export_matches_checked_in_artifact_and_format(): contract = export_guard_contract(CHAT_COMPLETIONS_ENDPOINT) assert contract == yaml.safe_load(EXPORTED.read_text(encoding="utf-8")) schema = json.loads((CONTRACTS / "guard-contract.schema.json").read_text(encoding="utf-8")) Draft202012Validator(schema).validate(contract) assert contract["version"] == CONTRACT_VERSION == "1.0.0-alpha.1" - assert "stream" not in contract + assert "stream" in contract assert "stream_hooks" not in contract["integration"]["endpoint"] diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py index d9471b86b9..584fa12ccd 100644 --- a/tests/server/experimental/test_openai_chat_streaming_integration.py +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -430,10 +430,10 @@ async def stream_dispatch(_request): def test_openai_endpoint_creates_fresh_stream_adapters(): - assert CHAT_COMPLETIONS_ENDPOINT.stream_adapter_factory is not None + assert CHAT_COMPLETIONS_ENDPOINT.stream is not None - first = CHAT_COMPLETIONS_ENDPOINT.stream_adapter_factory() - second = CHAT_COMPLETIONS_ENDPOINT.stream_adapter_factory() + first = CHAT_COMPLETIONS_ENDPOINT.stream.create_adapter() + second = CHAT_COMPLETIONS_ENDPOINT.stream.create_adapter() assert first is not second assert first.hooks is not second.hooks diff --git a/tests/server/experimental/test_openai_chat_streaming_projections.py b/tests/server/experimental/test_openai_chat_streaming_projections.py index 5dec6dffec..e53bbe0a48 100644 --- a/tests/server/experimental/test_openai_chat_streaming_projections.py +++ b/tests/server/experimental/test_openai_chat_streaming_projections.py @@ -35,9 +35,9 @@ from nemoguardrails.server.experimental.provider.sse import ServerSentEvent from nemoguardrails.server.experimental.provider.stream import ( ClassifiedStreamAdapter, + StreamBinding, StreamEventRole, UnsupportedProviderStream, - create_classified_stream_adapter_factory, ) from nemoguardrails.server.experimental.provider.types import GuardedMessage, UnknownContentFieldPolicy from nemoguardrails.server.experimental.providers.openai.chat_completions.request_binding import ( @@ -361,9 +361,9 @@ def test_stream_hooks_reject_payloads_after_provider_error(tail): observe(hooks, tail) -def test_stream_adapter_factory_creates_independent_completion_state(): - factory = create_classified_stream_adapter_factory(STREAM_CLASSIFIER, ChatCompletionsStreamHooks) - first, second = factory(), factory() +def test_stream_binding_creates_independent_completion_state(): + binding = StreamBinding(STREAM_CLASSIFIER, ChatCompletionsStreamHooks) + first, second = binding.create_adapter(), binding.create_adapter() first.classify_event(event(b"[DONE]")) second.classify_event(event(chunk({"content": "safe"}))) diff --git a/tests/server/experimental/test_stream_contract_export.py b/tests/server/experimental/test_stream_contract_export.py new file mode 100644 index 0000000000..07ee6f68c6 --- /dev/null +++ b/tests/server/experimental/test_stream_contract_export.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Verify that stream export describes the binding without constructing hooks.""" + +from dataclasses import replace + +import pytest + +from nemoguardrails.server.experimental.provider.contract_export import export_guard_contract +from nemoguardrails.server.experimental.provider.projection_policy import EXTENSION +from nemoguardrails.server.experimental.provider.sse import ServerSentEvent +from nemoguardrails.server.experimental.provider.stream import StreamBinding, StreamEventRole +from nemoguardrails.server.experimental.provider.stream_classifier import StreamEventRule +from nemoguardrails.server.experimental.provider.stream_policy import stream_shape_coverage +from nemoguardrails.server.experimental.providers.openai.chat_completions.endpoint import CHAT_COMPLETIONS_ENDPOINT +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_classifier import STREAM_CLASSIFIER +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_hooks import ChatCompletionsStreamHooks + + +def export_stream(classifier=STREAM_CLASSIFIER): + binding = StreamBinding(classifier, ChatCompletionsStreamHooks) + return export_guard_contract(replace(CHAT_COMPLETIONS_ENDPOINT, stream=binding))["stream"] + + +def with_rules(*rules, **transport): + classifier = replace(STREAM_CLASSIFIER, rules=rules, **transport) + coverage = stream_shape_coverage(rules, sentinels=classifier.sentinels, non_data_shape=classifier.non_data_shape) + return replace(classifier, contract=replace(classifier.contract, shapes=coverage, fields=None)) + + +def test_stream_export_describes_actual_rules_and_transport(): + stream = export_stream() + assert stream["title"] == STREAM_CLASSIFIER.contract.projection_id + assert stream[EXTENSION]["transport"] == { + "require_sse_event": False, + "non_data_shape": "keepalive", + "sentinels": {"[DONE]": "[DONE]"}, + } + text, error = stream["oneOf"] + assert text[EXTENSION]["event"] == { + "classification": "guarded_delta", + "variants": [ + { + "source_schema": "CreateChatCompletionStreamResponse", + "shape": "chat.completion.chunk:content", + "match": {"object": "chat.completion.chunk"}, + } + ], + "missing_text": "opaque", + "missing_text_shape": "chat.completion.chunk:metadata", + } + assert error[EXTENSION]["event"] == { + "classification": "provider_error", + "variants": [{"source_schema": "ErrorResponse", "shape": "error", "required_fields": ["error"]}], + } + choices = text["properties"]["choices"] + assert choices["maxItems"] == 1 + delta = choices["items"]["properties"]["delta"] + assert delta["properties"]["content"][EXTENSION]["subject"]["role"] == "assistant" + assert choices["items"]["properties"]["logprobs"]["type"] == "null" + + +def test_buffered_only_export_preserves_payloads_and_labels(): + full = export_guard_contract(CHAT_COMPLETIONS_ENDPOINT) + buffered = export_guard_contract(replace(CHAT_COMPLETIONS_ENDPOINT, stream=None)) + assert buffered == {key: value for key, value in full.items() if key != "stream"} + + +def test_binding_and_export_do_not_instantiate_hooks(): + def forbidden(): + raise AssertionError("Inspection must not construct protocol state") + + binding = StreamBinding(STREAM_CLASSIFIER, forbidden) + endpoint = replace(CHAT_COMPLETIONS_ENDPOINT, stream=binding) + assert export_guard_contract(endpoint)["stream"] == export_stream() + with pytest.raises(AssertionError, match="protocol state"): + binding.create_adapter() + + +def test_manual_classifier_is_runtime_only(): + class ManualClassifier: + contract = STREAM_CLASSIFIER.contract + + def classify_event(self, event): + return STREAM_CLASSIFIER.classify_event(event) + + binding = StreamBinding(ManualClassifier(), ChatCompletionsStreamHooks) + endpoint = replace(CHAT_COMPLETIONS_ENDPOINT, stream=binding) + event = ServerSentEvent.from_bytes(b"data: [DONE]\n\n") + adapter = binding.create_adapter() + assert adapter.classify_event(event).role is StreamEventRole.OPAQUE_METADATA + adapter.validate_end_of_stream() + with pytest.raises(TypeError, match="standard declarative"): + export_guard_contract(endpoint) + + +def test_binding_validates_hooks_only_when_creating_an_adapter(): + binding = StreamBinding(STREAM_CLASSIFIER, lambda: object()) + with pytest.raises(TypeError, match="provider stream hooks"): + binding.create_adapter() + + +@pytest.mark.parametrize("classifier,hooks", [(object(), ChatCompletionsStreamHooks), (STREAM_CLASSIFIER, None)]) +def test_binding_rejects_invalid_declarations(classifier, hooks): + with pytest.raises(TypeError, match="stream binding"): + StreamBinding(classifier, hooks) + + +def test_endpoint_requires_an_explicit_stream_binding(): + with pytest.raises(TypeError, match="StreamBinding"): + replace(CHAT_COMPLETIONS_ENDPOINT, stream=lambda: None) + + +@pytest.mark.parametrize( + "changes,message", + [ + ({"text_path": ("object",)}, "text binding differs"), + ({"text_path": None}, "inconsistent text role"), + ({"missing_text_value": "invisible"}, "must carry text"), + ], +) +def test_export_revalidates_bound_rules(changes, message): + rule = replace(STREAM_CLASSIFIER.rules[0], **changes) + classifier = replace(STREAM_CLASSIFIER, rules=(rule, STREAM_CLASSIFIER.rules[1])) + with pytest.raises(ValueError, match=message): + export_stream(classifier) + + +def test_export_rejects_shape_inventory_drift(): + classifier = replace(STREAM_CLASSIFIER, rules=STREAM_CLASSIFIER.rules[:1]) + with pytest.raises(ValueError, match="coverage differs"): + export_stream(classifier) + + +@pytest.mark.parametrize("value", [None, 0.0]) +def test_export_rejects_unrepresentable_match_values(value): + rule = replace(STREAM_CLASSIFIER.rules[0], match=(("object", value),)) + with pytest.raises(ValueError, match="unique match keys"): + export_stream(with_rules(rule)) + + +def test_export_rejects_duplicate_match_keys(): + rule = replace(STREAM_CLASSIFIER.rules[0], match=(("object", "one"), ("object", "two"))) + with pytest.raises(ValueError, match="unique match keys"): + export_stream(with_rules(rule)) + + +def test_export_rejects_custom_rule_matching(): + class CustomRule(StreamEventRule): + def matches(self, payload): + return True + + rule = CustomRule( + shape="error", + model=STREAM_CLASSIFIER.rules[1].model, + role=StreamEventRole.PROVIDER_ERROR, + required_fields=frozenset({"error"}), + ) + with pytest.raises(TypeError, match="standard declarative event rules"): + export_stream(with_rules(rule)) + + +@pytest.mark.parametrize("role", [StreamEventRole.GUARDED_TEXT, StreamEventRole.TEXT_SNAPSHOT]) +@pytest.mark.parametrize("fallback", ["reject", "empty"]) +def test_export_describes_text_roles_and_supported_missing_text(role, fallback): + rule = replace( + STREAM_CLASSIFIER.rules[0], + role=role, + missing_text_role=role if fallback == "empty" else None, + missing_text_shape=None, + missing_text_value="" if fallback == "empty" else None, + ) + exported = export_stream(with_rules(rule))["oneOf"][0][EXTENSION]["event"] + assert exported["classification"] == ("snapshot" if role is StreamEventRole.TEXT_SNAPSHOT else "guarded_delta") + assert exported["missing_text"] == fallback + assert "missing_text_shape" not in exported + + +def test_export_describes_opaque_event_rules(): + rule = replace(STREAM_CLASSIFIER.rules[1], role=StreamEventRole.OPAQUE_METADATA) + exported = export_stream(with_rules(rule))["oneOf"][0][EXTENSION]["event"] + assert exported["classification"] == "opaque" + assert "missing_text" not in exported + + +@pytest.mark.parametrize( + "role,value", + [ + (StreamEventRole.GUARDED_TEXT, "invented"), + (StreamEventRole.TEXT_SNAPSHOT, ""), + (StreamEventRole.PROVIDER_ERROR, None), + ], +) +def test_export_rejects_lossy_missing_text_fallback(role, value): + rule = replace( + STREAM_CLASSIFIER.rules[0], missing_text_role=role, missing_text_shape="fallback", missing_text_value=value + ) + with pytest.raises(ValueError, match="missing-text fallback"): + export_stream(with_rules(rule)) + + +def test_export_requires_explicit_source_provenance(monkeypatch): + model = STREAM_CLASSIFIER.rules[0].model + monkeypatch.setattr(model, "policy", replace(model.policy, source=None)) + with pytest.raises(ValueError, match="explicit policy source"): + export_stream() + + +@pytest.mark.parametrize("sentinels", [((b" [DONE]", "[DONE]"),), ((b"[DONE]", "[DONE]"), (b"[DONE]", "[DONE]"))]) +def test_export_rejects_noncanonical_or_duplicate_sentinels(sentinels): + with pytest.raises(ValueError, match="unique nonempty canonical"): + export_stream(replace(STREAM_CLASSIFIER, sentinels=sentinels)) + + +def test_export_rejects_binary_sentinels(): + with pytest.raises(ValueError, match="UTF-8"): + export_stream(replace(STREAM_CLASSIFIER, sentinels=((b"\xff", "[DONE]"),))) + + +def test_export_describes_data_discriminator_and_required_event(): + classifier = replace(STREAM_CLASSIFIER, event_type_field="object", require_event_type=True) + transport = export_stream(classifier)[EXTENSION]["transport"] + assert transport["require_sse_event"] is True + assert transport["data_discriminator"] == "object" + + +def test_stream_exports_do_not_share_mutable_state(): + exported = export_stream() + exported[EXTENSION]["transport"]["sentinels"].clear() + exported["oneOf"][0]["properties"].clear() + assert export_stream()[EXTENSION]["transport"]["sentinels"] == {"[DONE]": "[DONE]"} + assert "choices" in export_stream()["oneOf"][0]["properties"] From 2a77c38562883da542ac04051c3958b506d3d2b2 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:25:04 +0200 Subject: [PATCH 06/13] docs(server): narrow guarded content and export claims Empty assistant content means no guarded text because other reviewed fields that could carry generated content are rejected unless null or empty, not because every other field is content-free: opaque metadata is relayed without inspection. Export checks that replacement blockers name reviewed fields, not replacement behavior. Stream event match keys are dot-separated member paths, while object variant keys name item members directly. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- nemoguardrails/server/experimental/_guarded_proxy.py | 5 +++-- nemoguardrails/server/experimental/contracts/README.md | 9 +++++++-- .../server/experimental/contracts/openai/README.md | 6 +++--- nemoguardrails/server/experimental/provider/payload.py | 4 ++-- .../providers/openai/chat_completions/README.md | 7 ++++--- .../openai/chat_completions/response_projection.py | 5 +++-- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/nemoguardrails/server/experimental/_guarded_proxy.py b/nemoguardrails/server/experimental/_guarded_proxy.py index 2391c04a73..5e88564caa 100644 --- a/nemoguardrails/server/experimental/_guarded_proxy.py +++ b/nemoguardrails/server/experimental/_guarded_proxy.py @@ -153,8 +153,9 @@ def project_response( document = parse_json_object(payload.body) projection = endpoint.guarded_response_model.validate_payload(document) target = projection.locate_guarded_message(document) - # Validation has already rejected content in any other reviewed - # field, so a response without text has nothing to inspect. + # Validation has already rejected generated content in other + # reviewed fields, and opaque metadata is not inspected, so a + # response without text has no guarded text to check. if not target.has_text: return ContentInspectionNotApplicable() return target.message diff --git a/nemoguardrails/server/experimental/contracts/README.md b/nemoguardrails/server/experimental/contracts/README.md index 98b84e823f..806f2b89bc 100644 --- a/nemoguardrails/server/experimental/contracts/README.md +++ b/nemoguardrails/server/experimental/contracts/README.md @@ -92,8 +92,13 @@ cardinality. This is distinct from ordinary array length constraints. A stream's `oneOf` branches describe event families: `guarded_delta`, `snapshot`, `opaque`, or `provider_error`. Their variants identify source schemas, matching -rules, and classifier shape names. Missing-text policy is distinct from schema -nullability. Transport metadata describes SSE discriminators and sentinels. +rules, and classifier shape names. An event variant `match` key is a +dot-separated member path from the event payload root, such as `error.type`, +so a provider member whose name contains a dot cannot be matched. An object +`variant` `match` key names a member of the candidate item directly. Both +compare the exact parsed type and value, so `1`, `1.0`, and `true` are +distinct. Missing-text policy is distinct from schema nullability. Transport +metadata describes SSE discriminators and sentinels. Event exclusions and external-source reasons record scope and provenance; they do not establish that every provider event has been reviewed. diff --git a/nemoguardrails/server/experimental/contracts/openai/README.md b/nemoguardrails/server/experimental/contracts/openai/README.md index 6b1506b54b..daab4f0ec5 100644 --- a/nemoguardrails/server/experimental/contracts/openai/README.md +++ b/nemoguardrails/server/experimental/contracts/openai/README.md @@ -10,8 +10,8 @@ behavior. See [Guard contracts](../README.md) for the document vocabulary. | Area | Single-text boundary | | --- | --- | | Request | One user message with non-empty string content at `messages[0].content`. | -| Buffered response | One assistant choice with string content at `choices[0].message.content`. Empty or null content means the response has nothing to inspect, because every other reviewed field is content-free. The projection reports this through `has_text`, and the buffered integration relays such a response without output checks. | -| Streamed response | At most one choice, with integer index zero and optional assistant text at `choices[0].delta.content`. Missing or null text is metadata; an empty choices array allows usage chunks. Reviewed root metadata passes through. | +| Buffered response | One assistant choice with string content at `choices[0].message.content`. Empty or null content means the response has no guarded text: every other reviewed field that could carry generated content is rejected unless null or empty, and opaque metadata is not inspected. The projection reports this through `has_text`, and the buffered integration relays such a response without output checks. | +| Streamed response | At most one choice, with integer index zero and optional assistant text at `choices[0].delta.content`. Missing or null text is metadata; an empty choices array allows usage chunks. Reviewed root metadata passes through without inspection. | | Constrained values | User/assistant roles, single-item arrays, and request `n` constrained to one. | | Unsupported content | Tool, audio, multimodal, refusal, participant name, and separate reasoning content where explicitly disabled. | | Opaque data | Reviewed provider-owned metadata and controls, not additional guarded subjects. | @@ -22,7 +22,7 @@ messages alongside the user message, or multimodal content blocks. Disabled fields are null-only: a non-null value does not become acceptable merely because it requests plain text or no tools. A few unsupported features are constrained instead, because OpenAI's schema also allows a value that carries no content: -response `tool_calls` and `annotations` accept null or an empty list, and +buffered and stream `tool_calls` and response `annotations` accept null or an empty list, and request `logprobs` accepts null or `false`. Log probabilities are unsupported: they carry token text that rails do not diff --git a/nemoguardrails/server/experimental/provider/payload.py b/nemoguardrails/server/experimental/provider/payload.py index 9f5198658b..104ab2d9b7 100644 --- a/nemoguardrails/server/experimental/provider/payload.py +++ b/nemoguardrails/server/experimental/provider/payload.py @@ -199,8 +199,8 @@ def has_text(self) -> bool: """Return whether the target holds text for content checks. A validated projection reaches an empty or null target only when its - location allows empty text. The projection has then shown that no - other reviewed field carries content, so there is nothing to inspect. + location allows empty text. The projection has then rejected generated + content in other reviewed fields; opaque metadata is not inspected. """ content = self._object.get(self._member) return isinstance(content, str) and content != "" diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md index 88756d99f5..698f99d041 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md @@ -60,7 +60,8 @@ scope and provider provenance. Neither is loaded by the models or bindings. `ObjectPolicy` for the experimental `single_text_delta.v1` boundary. A chunk accepts zero or one choice with a strict integer index of zero, and `logprobs` must be null. The root, choice, and delta reject unreviewed members; reviewed -root metadata such as `id`, `model`, and `usage` passes through unchanged. +root metadata such as `id`, `model`, and `usage` passes through unchanged and +is not inspected. Missing or null content is metadata, while empty string content is still guarded text. Error messages are constrained fields, not guarded assistant output. The error envelope rejects unreviewed members, and its error object rejects them @@ -110,8 +111,8 @@ an injected streaming dispatcher; without one, they fail before dispatch. The shared [contract exporter](../../../provider/contract_export.py) takes the runtime endpoint, including its declared document identity. No per-operation exporter is needed. -Before export, it checks coverage, text bindings, replacement restrictions, and any -declared stream selector against the field policy. Stream rules are checked against +Before export, it checks coverage, text bindings, that replacement blockers name +reviewed fields, and any declared stream selector against the field policy. Stream rules are checked against typed text paths and exact shape coverage. Custom extraction, mismatched bindings, and unsupported event selectors or fallbacks are rejected rather than silently omitted. A handwritten classifier can still run through `StreamBinding`, but the diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/response_projection.py b/nemoguardrails/server/experimental/providers/openai/chat_completions/response_projection.py index d0cbf4a8c7..f30ec710ca 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/response_projection.py +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/response_projection.py @@ -33,8 +33,9 @@ class ChatCompletionsAssistantMessageProjection(PolicyModel, GuardedContentModel policy: ClassVar[ObjectPolicy] = ObjectPolicy(source="ChatCompletionResponseMessage", unknown_fields="configurable") # OpenAI allows empty or null content, for example when generation stops - # at a length limit or a content filter. Every other reviewed field is - # content-free here, so such a response has nothing to inspect. + # at a length limit or a content filter. Other reviewed fields that could + # carry generated content must then be null or empty, and opaque metadata + # is not inspected, so such a response has no guarded text. content: Annotated[ str | None, guarded( From ca4dafb57c26dc294c670930746c5190cefb8b59 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:17:23 +0200 Subject: [PATCH 07/13] fix(server): treat nullable Chat stream selectors as buffered Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/contracts/README.md | 5 ++- .../contracts/guard-contract.schema.json | 1 + .../_generated/chat-completions.guard.yaml | 4 +- .../experimental/provider/contract_export.py | 11 ++++- .../server/experimental/provider/payload.py | 6 ++- .../openai/chat_completions/README.md | 9 ++-- .../chat_completions/request_projection.py | 2 +- .../experimental/test_contract_export.py | 45 ++++++++++++++++++- .../test_openai_chat_contract_export.py | 15 +++++++ .../test_openai_chat_projections.py | 15 ++++--- .../test_openai_chat_streaming_integration.py | 10 ++++- 11 files changed, 104 insertions(+), 19 deletions(-) diff --git a/nemoguardrails/server/experimental/contracts/README.md b/nemoguardrails/server/experimental/contracts/README.md index 806f2b89bc..6b5e48581d 100644 --- a/nemoguardrails/server/experimental/contracts/README.md +++ b/nemoguardrails/server/experimental/contracts/README.md @@ -88,7 +88,10 @@ Provider-required opaque fields may be left to provider validation; a projection Object `variant` metadata describes discriminator matching and selected-variant cardinality. This is distinct from ordinary array length constraints. -`stream_selector_field` identifies a request flag, not streaming support. +`stream_selector_field` names a boolean request field: true selects streaming; +false or null selects buffered output, with null allowed only when the field's +schema permits it. Omission follows the declared default. This marker does not +enable streaming support or make a field nullable. A stream's `oneOf` branches describe event families: `guarded_delta`, `snapshot`, `opaque`, or `provider_error`. Their variants identify source schemas, matching diff --git a/nemoguardrails/server/experimental/contracts/guard-contract.schema.json b/nemoguardrails/server/experimental/contracts/guard-contract.schema.json index cf4921857a..98e3ee750b 100644 --- a/nemoguardrails/server/experimental/contracts/guard-contract.schema.json +++ b/nemoguardrails/server/experimental/contracts/guard-contract.schema.json @@ -361,6 +361,7 @@ ] }, "stream_selector_field": { + "description": "Names a boolean request field: true selects streaming; false or a schema-permitted null selects buffered output. Omission follows the field default. The marker does not enable streaming support.", "type": "string", "minLength": 1 }, diff --git a/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml b/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml index 44031228ee..c27586ae09 100644 --- a/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml +++ b/nemoguardrails/server/experimental/contracts/openai/_generated/chat-completions.guard.yaml @@ -176,9 +176,11 @@ request: classification: opaque stream: default: false - type: boolean x-nemo-guardrails: classification: constrained + oneOf: + - type: boolean + - type: 'null' stream_options: x-nemo-guardrails: classification: opaque diff --git a/nemoguardrails/server/experimental/provider/contract_export.py b/nemoguardrails/server/experimental/provider/contract_export.py index bd1d13ffa6..bdb21cc1c5 100644 --- a/nemoguardrails/server/experimental/provider/contract_export.py +++ b/nemoguardrails/server/experimental/provider/contract_export.py @@ -33,7 +33,8 @@ import json import sys from pathlib import Path -from typing import Any, Literal, get_args, get_origin +from types import UnionType +from typing import Annotated, Any, Literal, Union, get_args, get_origin from nemoguardrails.server.experimental.provider.endpoint import GuardedJsonEndpoint from nemoguardrails.server.experimental.provider.payload import ( @@ -150,12 +151,18 @@ def _validate_policy_binding( def _validate_stream_selector(model: type[GuardedRequestModel]) -> None: - """Require a declared selector to use the standard boolean-field binding.""" + """Require a boolean selector; permitted null values select buffered output.""" selector = model.stream_selector_field if selector is None: return field = model.model_fields.get(selector) annotation = field.annotation if field is not None else None + if get_origin(annotation) in (Union, UnionType) and type(None) in get_args(annotation): + members = [member for member in get_args(annotation) if member is not type(None)] + if len(members) == 1: + annotation = members[0] + while get_origin(annotation) is Annotated: + annotation = get_args(annotation)[0] literal_boolean = get_origin(annotation) is Literal and all(type(value) is bool for value in get_args(annotation)) if annotation is not bool and not literal_boolean: raise ValueError("The stream selector must name a boolean field") diff --git a/nemoguardrails/server/experimental/provider/payload.py b/nemoguardrails/server/experimental/provider/payload.py index 104ab2d9b7..2a332a3d71 100644 --- a/nemoguardrails/server/experimental/provider/payload.py +++ b/nemoguardrails/server/experimental/provider/payload.py @@ -388,7 +388,11 @@ def has_response_mode_binding(cls) -> bool: @property def streams_response(self) -> bool: - """Return the provider response mode selected by this request.""" + """Read the validated selector; a permitted null selects buffered output. + + Missing fields use their model default. The field declaration determines + whether null is accepted; the binding does not make a field nullable. + """ if self.stream_selector_field is None: raise TypeError("The guarded request model does not declare a stream selector field.") return bool(getattr(self, self.stream_selector_field)) diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md index 698f99d041..97fc8f9307 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md @@ -6,7 +6,7 @@ guarded/constrained/disabled/opaque policy, and assignments express defaults: ```python n: Annotated[Literal[1], BeforeValidator(_require_int), constrained(reason="core_capability.single_text_target")] = 1 -stream: Annotated[StrictBool, constrained()] = False +stream: Annotated[StrictBool | None, constrained()] = False audio: Annotated[None, disabled("core_capability.audio_content")] = None ``` @@ -37,9 +37,12 @@ validators. Streaming classification, stateful hooks, and endpoint construction are outside this buffered projection layer. -Existing buffered validation behavior is preserved: omitted disabled fields +Buffered validation uses the same field policy: omitted disabled fields default to null, explicit non-null disabled values fail, `stream` is a strict -boolean, and response annotations accept only null or an empty list. `n` accepts only the +boolean or null, and response annotations accept only null or an empty list. Only +`stream: true` selects streaming; false, null, or omission selects buffered output. +The original request bytes are forwarded without normalizing null to false. +`n` accepts only the integer `1`; booleans, strings, and floats are rejected. Generic JSON Schema and Python validation are not claimed to be interchangeable. Canonical exports are closed by default and explicitly declare configurable unknown-field handling diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/request_projection.py b/nemoguardrails/server/experimental/providers/openai/chat_completions/request_projection.py index 032300d8a5..1aad776be7 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/request_projection.py +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/request_projection.py @@ -86,7 +86,7 @@ class ChatCompletionsGuardedRequestProjection(PolicyModel, GuardedRequestModel): n: Annotated[ Literal[1], BeforeValidator(_require_int), constrained(reason="core_capability.single_text_target") ] = 1 - stream: Annotated[StrictBool, constrained()] = False + stream: Annotated[StrictBool | None, constrained()] = False audio: Annotated[None, disabled("core_capability.audio_content")] = None function_call: Annotated[None, disabled("core_capability.tool_content")] = None functions: Annotated[None, disabled("core_capability.tool_content")] = None diff --git a/tests/server/experimental/test_contract_export.py b/tests/server/experimental/test_contract_export.py index 2c9f3bb276..45f65e6787 100644 --- a/tests/server/experimental/test_contract_export.py +++ b/tests/server/experimental/test_contract_export.py @@ -21,10 +21,11 @@ import sys from dataclasses import replace from pathlib import Path -from typing import Annotated +from typing import Annotated, Literal import pytest from jsonschema import Draft202012Validator +from pydantic import StrictBool from nemoguardrails.server.experimental.provider.contract_export import _load_endpoint, export_guard_contract from nemoguardrails.server.experimental.provider.endpoint import GuardedJsonEndpoint @@ -154,6 +155,48 @@ def test_export_rejects_non_boolean_stream_selectors(example_endpoint, selector, export_guard_contract(example_endpoint) +@pytest.mark.parametrize( + ("annotation", "accepted"), + [ + (bool, True), + (StrictBool, True), + (bool | None, True), + (StrictBool | None, True), + (Literal[True, False] | None, True), + (str | None, False), + (int | None, False), + (bool | int | None, False), + (Literal[0, 1] | None, False), + (type(None), False), + ], +) +def test_export_accepts_only_boolean_or_nullable_boolean_selectors(example_endpoint, annotation, accepted): + """Export recognizes nullable booleans, not arbitrary truthy field types.""" + + class Request(PolicyModel, GuardedRequestModel): + prompt: Annotated[str, guarded("user")] + stream: Annotated[annotation, constrained()] = False + + Request.projection_contract = payload_contract(Request, projection_id="example.request", direction="request") + Request.guarded_text_location = text_location(Request) + Request.stream_selector_field = "stream" + endpoint = replace(example_endpoint, guarded_request_model=Request) + if not accepted: + with pytest.raises(ValueError, match="boolean field"): + export_guard_contract(endpoint) + return + contract = export_guard_contract(endpoint) + assert contract["request"][EXTENSION]["stream_selector_field"] == "stream" + validator = Draft202012Validator(contract["request"]) + for value in (True, False, None): + payload = {"prompt": "question", "stream": value} + if value is None and annotation in (bool, StrictBool): + assert not validator.is_valid(payload) + else: + validator.validate(payload) + assert Request.model_validate(payload).streams_response is (value is True) + + def test_export_rejects_custom_selection_with_a_declared_stream_selector(example_endpoint): """An exported boolean selector must describe the runtime selection rule.""" diff --git a/tests/server/experimental/test_openai_chat_contract_export.py b/tests/server/experimental/test_openai_chat_contract_export.py index 2f46e066bf..06fea2a59a 100644 --- a/tests/server/experimental/test_openai_chat_contract_export.py +++ b/tests/server/experimental/test_openai_chat_contract_export.py @@ -69,6 +69,21 @@ def test_export_uses_the_runtime_endpoint_metadata(): ) +@pytest.mark.parametrize("value", [True, False, None, 0, 1, "true", "false", [], {}]) +def test_stream_selector_export_matches_runtime(value): + """The published selector permits null but does not coerce other values.""" + payload = {"messages": [{"role": "user", "content": "question"}], "stream": value} + validator = Draft202012Validator(export_guard_contract(CHAT_COMPLETIONS_ENDPOINT)["request"]) + model = CHAT_COMPLETIONS_ENDPOINT.guarded_request_model + if value is None or type(value) is bool: + validator.validate(payload) + assert model.validate_payload(payload).streams_response is (value is True) + else: + assert not validator.is_valid(payload) + with pytest.raises(ValidationError): + model.validate_payload(payload) + + def test_export_does_not_read_files(monkeypatch): def unexpected_read(*args, **kwargs): raise AssertionError("Export must derive from Python, not read YAML") diff --git a/tests/server/experimental/test_openai_chat_projections.py b/tests/server/experimental/test_openai_chat_projections.py index 5a34903e8a..af8d1bd4bb 100644 --- a/tests/server/experimental/test_openai_chat_projections.py +++ b/tests/server/experimental/test_openai_chat_projections.py @@ -185,11 +185,12 @@ def test_request_projection_accepts_openai_reasoning_efforts(effort): assert projection.reasoning_effort == effort -def test_request_projection_reports_streaming_response_mode(): - """The request projection preserves the operation's boolean stream selector.""" - _, projection, _ = _guarded_request(_json_bytes(_request(stream=True))) +@pytest.mark.parametrize(("selector", "streaming"), [(True, True), (False, False), (None, False)]) +def test_request_projection_reports_streaming_response_mode(selector, streaming): + """Only true selects streaming; explicit null selects buffered output.""" + _, projection, _ = _guarded_request(_json_bytes(_request(stream=selector))) - assert projection.streams_response is True + assert projection.streams_response is streaming def test_response_binding_targets_original_provider_object_without_rewriting_bytes(): @@ -490,7 +491,7 @@ def test_n_rejects_coercible_and_other_values(value): ChatCompletionsGuardedRequest.model_validate({"messages": [{"role": "user", "content": "q"}], "n": value}) -@pytest.mark.parametrize("value", [0, 1, "true", None]) +@pytest.mark.parametrize("value", [0, 1, "true", "false", "", [], {}]) def test_stream_stays_strict(value): with pytest.raises(ValidationError): ChatCompletionsGuardedRequest.model_validate({"messages": [{"role": "user", "content": "q"}], "stream": value}) @@ -551,7 +552,7 @@ def test_export_follows_each_object_unknown_field_policy(): assert node[EXTENSION]["reject_case_aliases"] is True -def test_field_validation_matches_original_unannotated_declarations(): +def test_field_validation_matches_equivalent_unannotated_declarations(): from itertools import product from typing import Any @@ -559,7 +560,7 @@ def test_field_validation_matches_original_unannotated_declarations(): class OriginalRequestFields(BaseModel): n: Literal[1] = 1 - stream: StrictBool = False + stream: StrictBool | None = False audio: None = None class OriginalResponseFields(BaseModel): diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py index 584fa12ccd..d9bce2d486 100644 --- a/tests/server/experimental/test_openai_chat_streaming_integration.py +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -204,7 +204,8 @@ async def stream_dispatch(request): @pytest.mark.asyncio -async def test_non_streaming_request_uses_the_same_operation_and_buffered_dispatch(): +@pytest.mark.parametrize("selector", [False, None, "omitted"]) +async def test_non_streaming_request_uses_the_same_operation_and_buffered_dispatch(selector): buffered = [] streamed = [] @@ -221,11 +222,16 @@ async def stream_dispatch(request): raise AssertionError("a buffered request must not use streaming dispatch") checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + payload = json.loads(request_body(stream=selector)) + if selector == "omitted": + del payload["stream"] + body = json.dumps(payload, indent=2).encode() async with make_client(checker, stream_dispatch, buffered_dispatch) as client: - response = await post_stream(client, request_body(stream=False)) + response = await post_stream(client, body) assert response.status_code == 200 assert len(buffered) == 1 + assert buffered[0].body == body assert streamed == [] assert [kind for kind, _ in checker.calls] == ["input", "output"] assert checker.policy_reads == 1 From 224f697d93c01fff4961b4a5774d6d27005d7328 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:25:34 +0200 Subject: [PATCH 08/13] fix(server): reject lossy stream policy exports and document ownership Export could describe a policy different from the one bound at runtime: - stream models with custom validators, whose effect the document omits; - a hand-written response-mode property, exported without any selector; - a non-strict stream selector, which accepts "true" while the document says boolean; - schema keywords the guard format cannot carry. Each is now rejected at export. The validator check reads Pydantic's core schema; a test pins every validator kind so a Pydantic change fails loudly. Strictness has no JSON Schema keyword and stays a documented limitation, not an export failure. Endpoints whose response mode is fixed by the route can no longer be exported until the format can describe them. The export also drops a selector check that can no longer be false, adds an HTTP test that the checker sees the original stream text, and documents resource ownership and intentional limits. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/contracts/README.md | 12 +++ .../experimental/provider/contract_export.py | 74 ++++++++++++---- .../provider/projection_policy.py | 30 +++++++ .../openai/chat_completions/README.md | 39 +++++++-- .../experimental/test_contract_export.py | 85 ++++++++++++++----- .../experimental/test_openai_chat_contract.py | 15 +++- .../test_openai_chat_streaming_integration.py | 37 ++++++++ .../test_stream_contract_export.py | 52 +++++++++++- 8 files changed, 297 insertions(+), 47 deletions(-) diff --git a/nemoguardrails/server/experimental/contracts/README.md b/nemoguardrails/server/experimental/contracts/README.md index 6b5e48581d..a86bf0be68 100644 --- a/nemoguardrails/server/experimental/contracts/README.md +++ b/nemoguardrails/server/experimental/contracts/README.md @@ -92,6 +92,15 @@ cardinality. This is distinct from ordinary array length constraints. false or null selects buffered output, with null allowed only when the field's schema permits it. Omission follows the declared default. This marker does not enable streaming support or make a field nullable. +The exporter requires a strict boolean selector with a literal default (or a +required field). Custom mode-selection properties, selector validators, and +default factories are runtime-only, even when no selector is named. + +Numeric restrictions use the standard `minimum` and `maximum` assertions. +Python strictness has no JSON Schema keyword and is a stated limitation, not an +export failure: JSON Schema cannot distinguish an integer token from an +integral floating-point token. Runtime constraints do not have to be +exportable to construct a binding. A stream's `oneOf` branches describe event families: `guarded_delta`, `snapshot`, `opaque`, or `provider_error`. Their variants identify source schemas, matching @@ -102,6 +111,9 @@ so a provider member whose name contains a dot cannot be matched. An object compare the exact parsed type and value, so `1`, `1.0`, and `true` are distinct. Missing-text policy is distinct from schema nullability. Transport metadata describes SSE discriminators and sentinels. +The standard stream exporter rejects custom Python validators instead of +presenting their result as an equivalent declarative policy. Runtime stream +classification independently rejects validation that changes the original text. Event exclusions and external-source reasons record scope and provenance; they do not establish that every provider event has been reviewed. diff --git a/nemoguardrails/server/experimental/provider/contract_export.py b/nemoguardrails/server/experimental/provider/contract_export.py index bdb21cc1c5..0997b4073f 100644 --- a/nemoguardrails/server/experimental/provider/contract_export.py +++ b/nemoguardrails/server/experimental/provider/contract_export.py @@ -33,8 +33,9 @@ import json import sys from pathlib import Path -from types import UnionType -from typing import Annotated, Any, Literal, Union, get_args, get_origin +from typing import Any, Literal + +from pydantic import TypeAdapter from nemoguardrails.server.experimental.provider.endpoint import GuardedJsonEndpoint from nemoguardrails.server.experimental.provider.payload import ( @@ -48,6 +49,7 @@ EXTENSION, PolicyModel, export_payload_schema, + model_graph, payload_contract, text_location, ) @@ -60,12 +62,41 @@ from nemoguardrails.server.experimental.provider.stream_policy import build_policy_stream_classifier +def _validate_stream_model(model: type[PolicyModel]) -> None: + """Reject custom stream validation that cannot be described by field policy.""" + + def visit(node: Any) -> None: + if isinstance(node, dict): + kind = node.get("type") + if isinstance(kind, str) and kind in { + "function-before", + "function-after", + "function-wrap", + "function-plain", + }: + if node["function"]["function"] is not PolicyModel.reject_unreviewed_fields: + raise ValueError("Custom stream validators cannot be represented by the contract") + for name, child in node.items(): + if isinstance(kind, str) and name in {"metadata", "default", "config", "function"}: + continue + visit(child) + elif isinstance(node, (list, tuple)): + for child in node: + visit(child) + + for nested in model_graph(model).values(): + if nested.model_validate.__func__ is not PolicyModel.model_validate.__func__: + raise ValueError("Custom stream validators cannot be represented by the contract") + visit(model.__pydantic_core_schema__) + + def _export_stream_event(rule: StreamEventRule) -> dict[str, Any]: """Describe a typed event rule without dropping unsupported selection or fallback semantics.""" if type(rule) is not StreamEventRule: raise TypeError("Stream contract export requires standard declarative event rules") if not issubclass(rule.model, PolicyModel) or not rule.model.policy.source: raise ValueError("Stream event export requires an explicit policy source schema") + _validate_stream_model(rule.model) if len(dict(rule.match)) != len(rule.match) or any(type(value) not in (str, int, bool) for _, value in rule.match): raise ValueError("Stream event export requires unique match keys with string, integer, or boolean values") roles = { @@ -151,23 +182,31 @@ def _validate_policy_binding( def _validate_stream_selector(model: type[GuardedRequestModel]) -> None: - """Require a boolean selector; permitted null values select buffered output.""" + """Require declarative strict-boolean selection with a representable default.""" + if model.streams_response is not GuardedRequestModel.streams_response: + raise ValueError("Custom response-mode selection cannot be represented by the declared stream selector") selector = model.stream_selector_field if selector is None: - return + raise ValueError("The stream selector must name a strict boolean field") field = model.model_fields.get(selector) - annotation = field.annotation if field is not None else None - if get_origin(annotation) in (Union, UnionType) and type(None) in get_args(annotation): - members = [member for member in get_args(annotation) if member is not type(None)] - if len(members) == 1: - annotation = members[0] - while get_origin(annotation) is Annotated: - annotation = get_args(annotation)[0] - literal_boolean = get_origin(annotation) is Literal and all(type(value) is bool for value in get_args(annotation)) - if annotation is not bool and not literal_boolean: - raise ValueError("The stream selector must name a boolean field") - if model.streams_response is not GuardedRequestModel.streams_response: - raise ValueError("Custom response-mode selection cannot be represented by the declared stream selector") + if field is None: + raise ValueError("The stream selector must name a strict boolean field") + adapter = TypeAdapter(field.rebuild_annotation(), config={"strict": model.model_config.get("strict", False)}) + schema = adapter.core_schema + if schema["type"] == "nullable": + schema = schema["schema"] + if schema["type"] != "bool" or not schema.get("strict", model.model_config.get("strict", False)): + raise ValueError("The stream selector must name a strict boolean field") + decorators = model.__pydantic_decorators__ + if any(selector in item.info.fields or "*" in item.info.fields for item in decorators.field_validators.values()): + raise ValueError("Custom stream selector validators cannot be represented by the contract") + allowed = {PolicyModel.reject_unreviewed_fields, GuardedPayloadModel.validate_guarded_text_target} + if any(item.func not in allowed for item in decorators.model_validators.values()): + raise ValueError("Custom response-mode validation cannot be represented by the declared stream selector") + if field.default_factory is not None: + raise ValueError("A stream selector default factory cannot be represented by the contract") + if not field.is_required(): + adapter.validate_python(field.default) def export_guard_contract(endpoint: GuardedJsonEndpoint) -> dict[str, Any]: @@ -204,8 +243,7 @@ def export_guard_contract(endpoint: GuardedJsonEndpoint) -> dict[str, Any]: _validate_stream_selector(request_model) request = export_payload_schema(request_model, projection_id=request_contract.projection_id) response = export_payload_schema(response_model, projection_id=response_contract.projection_id) - if request_model.stream_selector_field is not None: - request[EXTENSION]["stream_selector_field"] = request_model.stream_selector_field + request[EXTENSION]["stream_selector_field"] = request_model.stream_selector_field integration: dict[str, Any] = {} if endpoint.contract_name is not None: integration["name"] = endpoint.contract_name diff --git a/nemoguardrails/server/experimental/provider/projection_policy.py b/nemoguardrails/server/experimental/provider/projection_policy.py index 3860942635..3761fec6ca 100644 --- a/nemoguardrails/server/experimental/provider/projection_policy.py +++ b/nemoguardrails/server/experimental/provider/projection_policy.py @@ -433,6 +433,36 @@ def export_payload_schema(model: type[PolicyModel], *, projection_id: str) -> di def expand(node: dict[str, Any]) -> dict[str, Any]: """Normalize the inline schema to the narrower contract vocabulary.""" + supported = { + "title", + "description", + "$comment", + "examples", + "deprecated", + "readOnly", + "writeOnly", + "type", + "const", + "default", + "enum", + "minItems", + "maxItems", + "minLength", + "maxLength", + "pattern", + "minimum", + "maximum", + "properties", + "required", + "additionalProperties", + "items", + "anyOf", + "oneOf", + EXTENSION, + } + unsupported = {key for key in node if key not in supported and not key.startswith("x-")} + if unsupported: + raise ValueError(f"Schema keywords not expressible in the guard contract: {sorted(unsupported)}") result = dict(node) metadata = dict(result.get(EXTENSION, {})) if "properties" in result: diff --git a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md index 97fc8f9307..cc9b14a8d7 100644 --- a/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md +++ b/nemoguardrails/server/experimental/providers/openai/chat_completions/README.md @@ -74,12 +74,12 @@ always validates with the closed default. [stream_classifier.py](stream_classifier.py) owns explicit `StreamEventRule` declarations for content and provider errors, along with the opaque `[DONE]` sentinel. Field coverage comes from the models; shape coverage comes from event -rules. Data-less SSE events have a distinct `keepalive` shape; they are not +rules, missing-text fallbacks, and transport declarations. +Data-less SSE events have a distinct `keepalive` shape; they are not terminal `[DONE]` events. Numeric bounds and strictness use ordinary Pydantic constraints, independent of the inspection document's supported vocabulary. Unsupported document representations fail at export, not runtime construction. -Shape coverage includes -rules, missing-text fallbacks, and transport declarations. The shared +The shared [stream policy helpers](../../../provider/stream_policy.py) derive a text path from typed fields, without schema generation, only when there is one subject and every traversed array has at most one item. A null container or null text @@ -93,8 +93,7 @@ cannot silently turn guarded text into a metadata-only event. The lower-level classifier resolves text from the original JSON before validation and rejects a projection that changes that text, including through container transformations. Bytes are relayed unchanged only after their original guarded text is checked. -The lower-level -classifier API remains available for handwritten bindings; those do not +The lower-level classifier API remains available for handwritten bindings; those do not automatically satisfy typed-policy validation or contract export. [stream_hooks.py](stream_hooks.py) remains handwritten. It enforces terminal @@ -120,6 +119,36 @@ typed text paths and exact shape coverage. Custom extraction, mismatched binding and unsupported event selectors or fallbacks are rejected rather than silently omitted. A handwritten classifier can still run through `StreamBinding`, but the exporter requires the standard declarative classifier. +Custom stream validators and custom response-mode properties are runtime-only, +not exported. This includes properties with no selector field. An exported +selector must be strict boolean (optionally nullable), with no default factory +or custom selector validation. Unsupported schema keywords fail at export. + +## Streaming ownership and limits + +Streaming dispatch owns the upstream resource until it returns +`StreamingHttpResponse`. A dispatcher using an async generator must provide +`close=upstream_response.aclose` (or an equivalent resource-owner callback), +not just the generator's `aclose`. Cancellation can permanently interrupt a +generator's awaited `finally`. An ordinary self-owning iterator may implement +idempotent `aclose` directly. If dispatch or response construction fails before +handoff, the dispatcher must close the resource itself. + +The HTTP response owns cleanup afterward, including disconnect before iteration, +source or checker failure, and normal EOF. Iterator cleanup and resource closing +use the same shielded, five-second-per-attempt helper. Closing can be retried; +callbacks must be idempotent and cooperate with cancellation. Timeouts and +ordinary close failures are logged without replacing the primary stream outcome. +This cannot forcibly close uncooperative code or guarantee resources were freed. + +With output inspection disabled, bytes are relayed without event classification +or provider lifecycle checks. Inspected streams check windows of nonempty text +deltas: `chunk_size` and `context_size` are counts, not tokens or bytes. Previously +released bytes cannot be retracted. Pending-byte limits do not include retained +context and are not a complete memory budget. The final pending events wait for +completion validation; upstream idle/read deadlines belong to transport policy. + +## Exporting the bound contract ```python from nemoguardrails.server.experimental.provider.contract_export import export_guard_contract diff --git a/tests/server/experimental/test_contract_export.py b/tests/server/experimental/test_contract_export.py index 45f65e6787..6f493276b2 100644 --- a/tests/server/experimental/test_contract_export.py +++ b/tests/server/experimental/test_contract_export.py @@ -21,7 +21,7 @@ import sys from dataclasses import replace from pathlib import Path -from typing import Annotated, Literal +from typing import Annotated, ClassVar, Literal import pytest from jsonschema import Draft202012Validator @@ -54,11 +54,8 @@ def example_endpoint() -> GuardedJsonEndpoint: class Request(PolicyModel, GuardedRequestModel): prompt: Annotated[str, guarded("user", replaceable=True, min_length=1)] - - @property - def streams_response(self) -> bool: - """This example has no request flag and always returns buffered JSON.""" - return False + stream: Annotated[StrictBool, constrained()] = False + stream_selector_field: ClassVar[str | None] = "stream" class Response(PolicyModel, GuardedPayloadModel): answer: Annotated[str, guarded("assistant", replaceable=False, min_length=2)] @@ -79,7 +76,7 @@ class Response(PolicyModel, GuardedPayloadModel): ) -def test_export_supports_another_endpoint_without_a_stream_selector(example_endpoint): +def test_export_supports_another_endpoint_with_a_buffered_default(example_endpoint): """The exporter derives both payloads and labels from the supplied endpoint.""" contract = export_guard_contract(example_endpoint) schema = json.loads( @@ -96,10 +93,10 @@ def test_export_supports_another_endpoint_without_a_stream_selector(example_endp "unsupported_response_code": "unsupported_example_response", } assert contract["request"]["title"] == "example.request" - assert set(contract["request"]["properties"]) == {"prompt"} + assert set(contract["request"]["properties"]) == {"prompt", "stream"} assert contract["response"]["title"] == "example.response" assert set(contract["response"]["properties"]) == {"answer"} - assert "stream_selector_field" not in contract["request"][EXTENSION] + assert contract["request"][EXTENSION]["stream_selector_field"] == "stream" assert "stream" not in contract @@ -115,7 +112,11 @@ def test_export_rejects_binding_policy_drift(example_endpoint, direction, mutati ) elif mutation == "coverage": contract = model.projection_contract - coverage = replace(contract.root, guarded_fields=frozenset(), constrained_fields=contract.root.guarded_fields) + coverage = replace( + contract.root, + guarded_fields=frozenset(), + constrained_fields=contract.root.constrained_fields | contract.root.guarded_fields, + ) monkeypatch.setattr(model, "projection_contract", replace(contract, root=coverage)) else: @@ -147,7 +148,7 @@ def streams_response(self) -> bool: export_guard_contract(endpoint) -@pytest.mark.parametrize("selector", ["missing", "prompt"]) +@pytest.mark.parametrize("selector", [None, "missing", "prompt"]) def test_export_rejects_non_boolean_stream_selectors(example_endpoint, selector, monkeypatch): """A declared stream selector must name a boolean field, not arbitrary metadata.""" monkeypatch.setattr(example_endpoint.guarded_request_model, "stream_selector_field", selector) @@ -158,11 +159,11 @@ def test_export_rejects_non_boolean_stream_selectors(example_endpoint, selector, @pytest.mark.parametrize( ("annotation", "accepted"), [ - (bool, True), + (bool, False), (StrictBool, True), - (bool | None, True), + (bool | None, False), (StrictBool | None, True), - (Literal[True, False] | None, True), + (Literal[True, False] | None, False), (str | None, False), (int | None, False), (bool | int | None, False), @@ -197,7 +198,8 @@ class Request(PolicyModel, GuardedRequestModel): assert Request.model_validate(payload).streams_response is (value is True) -def test_export_rejects_custom_selection_with_a_declared_stream_selector(example_endpoint): +@pytest.mark.parametrize("selector", [None, "stream"]) +def test_export_rejects_custom_selection_with_or_without_a_stream_selector(example_endpoint, selector): """An exported boolean selector must describe the runtime selection rule.""" class Request(PolicyModel, GuardedRequestModel): @@ -210,11 +212,56 @@ def streams_response(self) -> bool: Request.projection_contract = payload_contract(Request, projection_id="example.request", direction="request") Request.guarded_text_location = text_location(Request) - Request.stream_selector_field = "stream" + Request.stream_selector_field = selector with pytest.raises(ValueError, match="Custom response-mode selection"): export_guard_contract(replace(example_endpoint, guarded_request_model=Request)) +@pytest.mark.parametrize("default", [False, True, None]) +def test_export_preserves_nullable_selector_defaults(example_endpoint, default): + class Request(PolicyModel, GuardedRequestModel): + prompt: Annotated[str, guarded("user")] + stream: Annotated[StrictBool | None, constrained()] = default + stream_selector_field: ClassVar[str | None] = "stream" + + Request.projection_contract = payload_contract(Request, projection_id="example.request", direction="request") + Request.guarded_text_location = text_location(Request) + contract = export_guard_contract(replace(example_endpoint, guarded_request_model=Request)) + assert contract["request"]["properties"]["stream"]["default"] is default + assert Request.model_validate({"prompt": "question"}).streams_response is (default is True) + for value in ("true", "false", 0, 1): + with pytest.raises(ValueError): + Request.model_validate({"prompt": "question", "stream": value}) + + +def test_runtime_schema_constructs_fail_only_at_contract_export(example_endpoint): + from datetime import date + + class Request(example_endpoint.guarded_request_model): + date_value: Annotated[date, constrained()] + + Request.projection_contract = payload_contract(Request, projection_id="example.request", direction="request") + Request.model_validate({"prompt": "question", "date_value": "2026-10-12"}) + endpoint = replace(example_endpoint, guarded_request_model=Request) + with pytest.raises(ValueError, match="not expressible.*format"): + export_guard_contract(endpoint) + + +def test_selector_default_factory_is_not_silently_omitted(example_endpoint, monkeypatch): + field = example_endpoint.guarded_request_model.model_fields["stream"] + monkeypatch.setattr(field, "default_factory", lambda: True) + with pytest.raises(ValueError, match="default factory"): + export_guard_contract(example_endpoint) + + +@pytest.mark.parametrize("default", ["true", 1, None]) +def test_selector_defaults_must_match_the_declared_type(example_endpoint, monkeypatch, default): + field = example_endpoint.guarded_request_model.model_fields["stream"] + monkeypatch.setattr(field, "default", default) + with pytest.raises(ValueError): + export_guard_contract(example_endpoint) + + def test_export_follows_changed_endpoint_metadata(example_endpoint): """An endpoint variant changes the export without a provider-specific wrapper.""" endpoint = replace( @@ -243,10 +290,8 @@ def test_http_schema_does_not_require_exportable_unions(example_endpoint): class UnionRequest(PolicyModel, GuardedRequestModel): prompt: Annotated[str, guarded("user")] option: Annotated[str | int, constrained()] - - @property - def streams_response(self) -> bool: - return False + stream: Annotated[StrictBool, constrained()] = False + stream_selector_field: ClassVar[str | None] = "stream" UnionRequest.projection_contract = payload_contract( UnionRequest, projection_id="example.request", direction="request" diff --git a/tests/server/experimental/test_openai_chat_contract.py b/tests/server/experimental/test_openai_chat_contract.py index d4539ef7d6..44f3d8096c 100644 --- a/tests/server/experimental/test_openai_chat_contract.py +++ b/tests/server/experimental/test_openai_chat_contract.py @@ -132,10 +132,19 @@ def test_chat_exports_conform_to_the_guard_contract_schema(): assert _contract_errors(ChatCompletionsGuardedRequestProjection, ChatCompletionsGuardedResponseProjection) == [] -@pytest.mark.parametrize(("bound", "exportable"), [({"ge": 0}, True), ({"le": 0}, True), ({"gt": 0}, False)]) -def test_contract_schema_supports_only_inclusive_numeric_bounds(bound, exportable): +@pytest.mark.parametrize("bound", [{"ge": 0}, {"le": 0}]) +def test_contract_schema_supports_inclusive_numeric_bounds(bound): class Response(PolicyModel): text: Annotated[str, guarded("assistant")] count: Annotated[int, Field(**bound), constrained()] - assert (_contract_errors(ChatCompletionsGuardedRequestProjection, Response) == []) is exportable + assert _contract_errors(ChatCompletionsGuardedRequestProjection, Response) == [] + + +def test_export_rejects_numeric_bounds_the_contract_cannot_carry(): + class Response(PolicyModel): + text: Annotated[str, guarded("assistant")] + count: Annotated[int, Field(gt=0), constrained()] + + with pytest.raises(ValueError, match="not expressible in the guard contract: \\['exclusiveMinimum'\\]"): + _contract_errors(ChatCompletionsGuardedRequestProjection, Response) diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py index d9bce2d486..0b3200c2ff 100644 --- a/tests/server/experimental/test_openai_chat_streaming_integration.py +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -19,6 +19,7 @@ import httpx import pytest from fastapi import FastAPI +from pydantic import model_validator from nemoguardrails.server.experimental._content_checker import ( ContentAllowed, @@ -28,9 +29,15 @@ ) from nemoguardrails.server.experimental._http_kernel import BufferedHttpResponse from nemoguardrails.server.experimental._streaming_http import StreamingHttpResponse +from nemoguardrails.server.experimental.provider.stream import StreamBinding from nemoguardrails.server.experimental.providers.openai.chat_completions.endpoint import ( CHAT_COMPLETIONS_ENDPOINT, ) +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_classifier import STREAM_CLASSIFIER +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_hooks import ChatCompletionsStreamHooks +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_projection import ( + ChatCompletionsStreamPayloadProjection, +) from nemoguardrails.server.experimental.providers.openai.errors import OPENAI_ERROR_MAPPING from nemoguardrails.server.experimental.providers.openai.integration import create_openai_chat_router @@ -151,6 +158,36 @@ async def post_stream(client, body=None, **kwargs): return await client.post("/v1/chat/completions", content=body or request_body(), headers=headers, **kwargs) +@pytest.mark.asyncio +async def test_transforming_projection_never_checks_different_text_then_relays_original(monkeypatch): + from nemoguardrails.server.experimental.providers.openai import integration + + class TransformingPayload(ChatCompletionsStreamPayloadProjection): + @model_validator(mode="before") + @classmethod + def transform(cls, value): + value["choices"][0]["delta"]["content"] = "safe" + return value + + rule = replace(STREAM_CLASSIFIER.rules[0], model=TransformingPayload) + classifier = replace(STREAM_CLASSIFIER, rules=(rule, STREAM_CLASSIFIER.rules[1])) + endpoint = replace(CHAT_COMPLETIONS_ENDPOINT, stream=StreamBinding(classifier, ChatCompletionsStreamHooks)) + monkeypatch.setattr(integration, "CHAT_COMPLETIONS_ENDPOINT", endpoint) + source = Source([stream_event("unsafe-original") + b"data: [DONE]\n\n"]) + + async def dispatch(_request): + return StreamingHttpResponse(200, ((b"content-type", b"text/event-stream"),), source) + + checker = StaticChecker(ContentInspectionPolicy(True, True, StreamBufferingPolicy(1, 0))) + async with make_client(checker, dispatch) as client: + response = await post_stream(client) + + assert b"unsafe-original" not in response.content + assert CHAT_COMPLETIONS_ENDPOINT.unsupported_response_code.encode() in response.content + assert not any(kind == "output" for kind, _ in checker.calls) + assert source.closed + + @pytest.mark.asyncio async def test_streaming_chat_uses_one_prepared_request_and_preserves_provider_sse_bytes(): events = [ diff --git a/tests/server/experimental/test_stream_contract_export.py b/tests/server/experimental/test_stream_contract_export.py index 07ee6f68c6..4335e4515d 100644 --- a/tests/server/experimental/test_stream_contract_export.py +++ b/tests/server/experimental/test_stream_contract_export.py @@ -16,11 +16,13 @@ """Verify that stream export describes the binding without constructing hooks.""" from dataclasses import replace +from typing import Annotated import pytest +from pydantic import AfterValidator, BeforeValidator, PlainValidator, WrapValidator, model_validator from nemoguardrails.server.experimental.provider.contract_export import export_guard_contract -from nemoguardrails.server.experimental.provider.projection_policy import EXTENSION +from nemoguardrails.server.experimental.provider.projection_policy import EXTENSION, guarded from nemoguardrails.server.experimental.provider.sse import ServerSentEvent from nemoguardrails.server.experimental.provider.stream import StreamBinding, StreamEventRole from nemoguardrails.server.experimental.provider.stream_classifier import StreamEventRule @@ -28,6 +30,11 @@ from nemoguardrails.server.experimental.providers.openai.chat_completions.endpoint import CHAT_COMPLETIONS_ENDPOINT from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_classifier import STREAM_CLASSIFIER from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_hooks import ChatCompletionsStreamHooks +from nemoguardrails.server.experimental.providers.openai.chat_completions.stream_projection import ( + ChatCompletionsStreamChoiceProjection, + ChatCompletionsStreamDeltaProjection, + ChatCompletionsStreamPayloadProjection, +) def export_stream(classifier=STREAM_CLASSIFIER): @@ -71,6 +78,49 @@ def test_stream_export_describes_actual_rules_and_transport(): delta = choices["items"]["properties"]["delta"] assert delta["properties"]["content"][EXTENSION]["subject"]["role"] == "assistant" assert choices["items"]["properties"]["logprobs"]["type"] == "null" + index = choices["items"]["properties"]["index"] + assert index["type"] == "integer" + assert index["minimum"] == index["maximum"] == 0 + + +def test_export_rejects_custom_model_validation(): + class TransformingPayload(ChatCompletionsStreamPayloadProjection): + @model_validator(mode="before") + @classmethod + def transform(cls, value): + value["choices"][0]["delta"]["content"] = "safe" + return value + + rule = replace(STREAM_CLASSIFIER.rules[0], model=TransformingPayload) + with pytest.raises(ValueError, match="Custom stream validators"): + export_stream(with_rules(rule)) + + +@pytest.mark.parametrize( + "validator", + [ + BeforeValidator(lambda value: value), + AfterValidator(lambda value: value), + WrapValidator(lambda value, handler: handler(value)), + PlainValidator(lambda value: value), + ], + ids=["before", "after", "wrap", "plain"], +) +def test_export_rejects_nested_annotated_validation(validator): + """Pin every Pydantic validator kind, so a core-schema rename fails loudly.""" + + class TransformingDelta(ChatCompletionsStreamDeltaProjection): + content: Annotated[str | None, validator, guarded("assistant")] = None + + class TransformingChoice(ChatCompletionsStreamChoiceProjection): + delta: Annotated[TransformingDelta, guarded()] + + class TransformingPayload(ChatCompletionsStreamPayloadProjection): + choices: Annotated[list[TransformingChoice], guarded(max_length=1)] + + rule = replace(STREAM_CLASSIFIER.rules[0], model=TransformingPayload) + with pytest.raises(ValueError, match="Custom stream validators"): + export_stream(with_rules(rule)) def test_buffered_only_export_preserves_payloads_and_labels(): From 71d98f995fd68752797c382424c88fc0e28d62a4 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:00:33 +0200 Subject: [PATCH 09/13] fix(server): reject unrepresentable stream default factories A stream selector default produced by a factory runs at validation time, but the exported document can only state a literal default. Export now rejects selector default factories instead of silently omitting their behavior. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/provider/contract_export.py | 2 ++ .../experimental/test_stream_contract_export.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/nemoguardrails/server/experimental/provider/contract_export.py b/nemoguardrails/server/experimental/provider/contract_export.py index 0997b4073f..ab7c56811c 100644 --- a/nemoguardrails/server/experimental/provider/contract_export.py +++ b/nemoguardrails/server/experimental/provider/contract_export.py @@ -87,6 +87,8 @@ def visit(node: Any) -> None: for nested in model_graph(model).values(): if nested.model_validate.__func__ is not PolicyModel.model_validate.__func__: raise ValueError("Custom stream validators cannot be represented by the contract") + if any(field.default_factory is not None for field in nested.model_fields.values()): + raise ValueError("Stream default factories cannot be represented by the contract") visit(model.__pydantic_core_schema__) diff --git a/tests/server/experimental/test_stream_contract_export.py b/tests/server/experimental/test_stream_contract_export.py index 4335e4515d..c8700ea25c 100644 --- a/tests/server/experimental/test_stream_contract_export.py +++ b/tests/server/experimental/test_stream_contract_export.py @@ -19,7 +19,7 @@ from typing import Annotated import pytest -from pydantic import AfterValidator, BeforeValidator, PlainValidator, WrapValidator, model_validator +from pydantic import AfterValidator, BeforeValidator, Field, PlainValidator, WrapValidator, model_validator from nemoguardrails.server.experimental.provider.contract_export import export_guard_contract from nemoguardrails.server.experimental.provider.projection_policy import EXTENSION, guarded @@ -123,6 +123,17 @@ class TransformingPayload(ChatCompletionsStreamPayloadProjection): export_stream(with_rules(rule)) +def test_export_rejects_stream_default_factories(): + class DefaultedPayload(ChatCompletionsStreamPayloadProjection): + choices: Annotated[list[ChatCompletionsStreamChoiceProjection], guarded(max_length=1)] = Field( + default_factory=list + ) + + rule = replace(STREAM_CLASSIFIER.rules[0], model=DefaultedPayload) + with pytest.raises(ValueError, match="default factories"): + export_stream(with_rules(rule)) + + def test_buffered_only_export_preserves_payloads_and_labels(): full = export_guard_contract(CHAT_COMPLETIONS_ENDPOINT) buffered = export_guard_contract(replace(CHAT_COMPLETIONS_ENDPOINT, stream=None)) From 4e89e5924b7a557b26a7e7a7797678599349f0b5 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:47:08 +0200 Subject: [PATCH 10/13] fix(server): give stream output checks the checked input message The streaming branch passed guardable.target.message as output-check context, which equals the message the input rail checked only because the request projection happens to return it. It now passes prepared_input.input_message, as the buffered path does. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- nemoguardrails/server/experimental/_guarded_proxy.py | 2 +- .../experimental/test_openai_chat_streaming_integration.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nemoguardrails/server/experimental/_guarded_proxy.py b/nemoguardrails/server/experimental/_guarded_proxy.py index 5e88564caa..8eecde1ee5 100644 --- a/nemoguardrails/server/experimental/_guarded_proxy.py +++ b/nemoguardrails/server/experimental/_guarded_proxy.py @@ -216,7 +216,7 @@ async def handle_prepared_request( dispatch=stream_dispatch, checker=prepared_input.resolved.checker, streaming_policy=streaming_policy, - input_message=guardable.target.message, + input_message=prepared_input.input_message, adapter=stream_binding.create_adapter(), render_outcome=render_stream_outcome, max_event_bytes=max_stream_event_bytes, diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py index 0b3200c2ff..db165b53dc 100644 --- a/tests/server/experimental/test_openai_chat_streaming_integration.py +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -238,6 +238,9 @@ async def stream_dispatch(request): ] assert checker.policy_reads == 1 assert source.closed is True + input_check = checker.calls[0][1] + # Output checks get the message the input rail actually checked, not a re-projection. + assert all(call.input_message is input_check.message for kind, call in checker.calls if kind == "output") @pytest.mark.asyncio From 172a3c93377adbf5acacbe6ebb0741e063107b8f Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:49:47 +0200 Subject: [PATCH 11/13] fix(server): reject stream dispatch without an endpoint stream binding An operation built with a streaming dispatcher but no endpoint stream binding failed each stream: true request with an AssertionError, which surfaced as a 500. The combination is a configuration error, so it is now rejected when the operation is created, and the request path no longer needs the assertion. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../server/experimental/_guarded_proxy.py | 9 +++++---- .../experimental/test_openai_chat_integration.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/nemoguardrails/server/experimental/_guarded_proxy.py b/nemoguardrails/server/experimental/_guarded_proxy.py index 8eecde1ee5..cac154328f 100644 --- a/nemoguardrails/server/experimental/_guarded_proxy.py +++ b/nemoguardrails/server/experimental/_guarded_proxy.py @@ -123,6 +123,10 @@ def create_guarded_http_operation( ) -> GuardedHttpOperation: """Create one buffered-and-streaming operation from an endpoint contract.""" + stream_binding = endpoint.stream + if stream_dispatch is not None and stream_binding is None: + raise ValueError("A streaming dispatcher requires an endpoint stream binding.") + def prepare_request(request: BufferedHttpRequest) -> GuardableProviderRequest: """Prepare one buffered HTTP request for guarded execution.""" return _prepare_guardable_request(request, endpoint) @@ -194,7 +198,7 @@ async def handle_prepared_request( guardable = prepared_input.request if not guardable.streaming: return None - if stream_dispatch is None: + if stream_dispatch is None or stream_binding is None: return render_stream_outcome( OperationProjectionFailed( InspectionStage.INPUT, @@ -208,9 +212,6 @@ async def handle_prepared_request( UnsupportedStreamInspection("Guarded output streaming requires a buffering policy.") ) ) - stream_binding = endpoint.stream - if stream_binding is None: - raise AssertionError("A streaming request requires an endpoint stream adapter.") return await execute_streaming_http( guardable.request, dispatch=stream_dispatch, diff --git a/tests/server/experimental/test_openai_chat_integration.py b/tests/server/experimental/test_openai_chat_integration.py index 8191c12526..ecb9332ee0 100644 --- a/tests/server/experimental/test_openai_chat_integration.py +++ b/tests/server/experimental/test_openai_chat_integration.py @@ -825,3 +825,17 @@ def test_http_failures_render_as_openai_errors(kind, status_code, error_type): assert body["error"]["type"] == error_type assert body["error"]["code"] == kind.value assert "internal detail" not in response.body.decode() + + +def test_stream_dispatch_requires_an_endpoint_stream_binding(): + async def stream_dispatch(_request): + raise AssertionError("must not dispatch") + + with pytest.raises(ValueError, match="requires an endpoint stream binding"): + create_guarded_http_operation( + replace(CHAT_COMPLETIONS_ENDPOINT, stream=None), + OPENAI_ERROR_MAPPING, + stream_dispatch=stream_dispatch, + max_stream_event_bytes=1024, + max_pending_stream_bytes=1024, + ) From 7d5eb0da79979931306a12a0f2f1e967f6bbe900 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:49:48 +0200 Subject: [PATCH 12/13] test(server): cover streaming integration and export rejections Adds cases for selector field and model validators, overridden model validation on stream models, unknown capability profiles, a stream binding without a callable classifier, and the native error for stream processing failures. No runtime behavior changes. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- .../experimental/test_contract_export.py | 31 ++++++++++++++++++- .../test_openai_chat_streaming_integration.py | 10 ++++++ .../test_stream_contract_export.py | 26 ++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/tests/server/experimental/test_contract_export.py b/tests/server/experimental/test_contract_export.py index 6f493276b2..3f13dd9360 100644 --- a/tests/server/experimental/test_contract_export.py +++ b/tests/server/experimental/test_contract_export.py @@ -25,7 +25,7 @@ import pytest from jsonschema import Draft202012Validator -from pydantic import StrictBool +from pydantic import StrictBool, field_validator, model_validator from nemoguardrails.server.experimental.provider.contract_export import _load_endpoint, export_guard_contract from nemoguardrails.server.experimental.provider.endpoint import GuardedJsonEndpoint @@ -481,3 +481,32 @@ class UnlocatedResponse(GuardedPayloadModel): replace(example_endpoint, guarded_request_model=ModelessRequest) with pytest.raises(ValueError, match="response projection must declare its guarded text location"): replace(example_endpoint, guarded_response_model=UnlocatedResponse) + + +@pytest.mark.parametrize("kind", ["field", "model"]) +def test_export_rejects_custom_selector_validation(example_endpoint, kind): + """Validators can change which mode a request selects, which the selector cannot describe.""" + + class Request(PolicyModel, GuardedRequestModel): + prompt: Annotated[str, guarded("user")] + stream: Annotated[StrictBool | None, constrained()] = False + stream_selector_field: ClassVar[str | None] = "stream" + + if kind == "field": + + @field_validator("stream") + @classmethod + def keep(cls, value): + return value + + else: + + @model_validator(mode="after") + def keep(self): + return self + + Request.projection_contract = payload_contract(Request, projection_id="example.request", direction="request") + Request.guarded_text_location = text_location(Request) + message = "selector validators" if kind == "field" else "response-mode validation" + with pytest.raises(ValueError, match=message): + export_guard_contract(replace(example_endpoint, guarded_request_model=Request)) diff --git a/tests/server/experimental/test_openai_chat_streaming_integration.py b/tests/server/experimental/test_openai_chat_streaming_integration.py index db165b53dc..8f0e30dae6 100644 --- a/tests/server/experimental/test_openai_chat_streaming_integration.py +++ b/tests/server/experimental/test_openai_chat_streaming_integration.py @@ -27,6 +27,7 @@ ContentInspectionPolicy, StreamBufferingPolicy, ) +from nemoguardrails.server.experimental._guarded_stream import StreamProcessingFailed from nemoguardrails.server.experimental._http_kernel import BufferedHttpResponse from nemoguardrails.server.experimental._streaming_http import StreamingHttpResponse from nemoguardrails.server.experimental.provider.stream import StreamBinding @@ -581,3 +582,12 @@ async def dispatch(_request): assert response.json()["error"]["code"] == "unsupported_chat_completions_response_shape" assert "x-provider-secret" not in response.headers assert source.closed is True + + +def test_stream_processing_failures_render_a_native_proxy_error(): + response = OPENAI_ERROR_MAPPING.renderer(StreamProcessingFailed(RuntimeError("private detail"))) + + assert response.status_code == 502 + body = json.loads(response.body) + assert body["error"]["code"] == "stream_processing_failed" + assert "private detail" not in response.body.decode() diff --git a/tests/server/experimental/test_stream_contract_export.py b/tests/server/experimental/test_stream_contract_export.py index c8700ea25c..c00621653c 100644 --- a/tests/server/experimental/test_stream_contract_export.py +++ b/tests/server/experimental/test_stream_contract_export.py @@ -304,3 +304,29 @@ def test_stream_exports_do_not_share_mutable_state(): exported["oneOf"][0]["properties"].clear() assert export_stream()[EXTENSION]["transport"]["sentinels"] == {"[DONE]": "[DONE]"} assert "choices" in export_stream()["oneOf"][0]["properties"] + + +def test_export_rejects_overridden_model_validation(): + class OverriddenPayload(ChatCompletionsStreamPayloadProjection): + @classmethod + def model_validate(cls, obj, **kwargs): + return super().model_validate(obj, **kwargs) + + rule = replace(STREAM_CLASSIFIER.rules[0], model=OverriddenPayload) + with pytest.raises(ValueError, match="Custom stream validators"): + export_stream(with_rules(rule)) + + +def test_export_rejects_unknown_capability_profiles(): + contract = replace(STREAM_CLASSIFIER.contract, profile="single_text_delta.v2") + with pytest.raises(ValueError, match="capability profile"): + export_stream(replace(STREAM_CLASSIFIER, contract=contract)) + + +def test_stream_binding_requires_a_callable_classifier(): + class NotCallable: + contract = STREAM_CLASSIFIER.contract + classify_event = None + + with pytest.raises(TypeError, match="callable classifier"): + StreamBinding(NotCallable(), ChatCompletionsStreamHooks) From a7f7e64854ec3caaebdc7429779d7f4472b6e5cc Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:32:16 +0200 Subject: [PATCH 13/13] fix(server): name the stream outcomes that carry a stage StreamInspectionUnsupported joins the stream outcome union here and has no stage, so ty rejected the fallback branch. Signed-off-by: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> --- nemoguardrails/server/experimental/_guarded_stream.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemoguardrails/server/experimental/_guarded_stream.py b/nemoguardrails/server/experimental/_guarded_stream.py index 1263c9b2ee..648fc62c25 100644 --- a/nemoguardrails/server/experimental/_guarded_stream.py +++ b/nemoguardrails/server/experimental/_guarded_stream.py @@ -190,7 +190,8 @@ def _log_outcome(outcome: StreamOutcome) -> None: failure = outcome.failure if isinstance(failure, ContentCheckFailed): failure = failure.cause - stage = "stream" if isinstance(outcome, StreamUpstreamFailed | StreamProcessingFailed) else outcome.stage.value + staged = OperationCheckFailed | OperationModificationUnsupported | OperationProjectionFailed + stage = outcome.stage.value if isinstance(outcome, staged) else "stream" log.warning("Guarded stream stopped: %s at %s stage (%s)", type(outcome).__name__, stage, type(failure).__name__)