chore(relay): Consolidate integration tests

.gitattributes

@@ -1,4 +1,4 @@
 tests/server/experimental/conformance/openai/chat_completions/fixtures/pinned.openapi.yaml text eol=lf
 tests/server/experimental/conformance/openai/responses/fixtures/source_inventory.json text eol=lf
 Cargo.lock linguist-generated=true
-tests_relay_plugin/requirements.txt linguist-generated=true
+integration_tests/relay_plugin/requirements.txt linguist-generated=true

AGENTS.md

@@ -59,6 +59,7 @@ skill discovery.
 
 - Main package: `nemoguardrails/`
 - Tests: `tests/` and `benchmark/tests` (the `testpaths` in `pytest.ini`)
+- End-to-end suites outside the `testpaths`: `integration_tests/` (see its `README.md`)
 - Schemas and validation snapshots: `schemas/`
 - Default development branch: `develop`
 

CONTRIBUTING.md

@@ -191,6 +191,9 @@ Run the smallest meaningful test set first, then broaden validation when the
 change touches shared runtime behavior, public APIs, packaging, server behavior,
 tracing, or docs.
 
+The suites in `integration_tests/` run NeMo Guardrails against real agents and clients, and `make test` doesn't run them.
+See [`integration_tests/README.md`](integration_tests/README.md) for what each suite checks, what it needs, and how to run it.
+
 Set up local pre-commit hooks if you want checks to run before every commit:
 
 ```bash

Makefile

@@ -28,14 +28,14 @@ UNIT_TEST_ENV ?= env -u OPENAI_API_KEY -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_KEY
 # directory of their own rather than .venv, so neither is a Guardrails dependency. `uv pip sync` makes the directory
 # match requirements.txt and checks every wheel against its hash. The tests run in .venv, with live provider keys unset
 # as for `make test`, and find the agents through HOOKS_AGENTS_DIR.
-HOOKS_AGENTS_TESTS ?= tests_hooks_agents
+HOOKS_AGENTS_TESTS ?= integration_tests/hooks_agents
 HOOKS_AGENTS_DIR ?= .cache/hooks_agents
 
 # The NeMo Relay plugin's end-to-end tests run in their own Python 3.11 environment without Guardrails, while the hooks
 # servers they start run from .venv. `uv pip sync --require-hashes` makes that environment match requirements.txt and
 # checks every package against its hash, which `uv run --with-requirements` doesn't. nemo-relay is exempt from
 # [tool.uv]'s exclude-newer window, which would filter out a recent pin. Live provider keys are unset as for `make test`.
-RELAY_PLUGIN_TESTS ?= tests_relay_plugin
+RELAY_PLUGIN_TESTS ?= integration_tests/relay_plugin
 RELAY_PLUGIN_VENV ?= .venv-relay-plugin
 RELAY_PLUGIN_PYTEST ?= $(UNIT_TEST_ENV) $(RELAY_PLUGIN_VENV)/bin/pytest
 
@@ -166,8 +166,8 @@ help:
 		'  make test-benchmark [ARGS="-q"]' \
 		'  make test-parallel [TEST=path] [WORKERS=auto] [ARGS="-q --tb=short"]' \
 		'  make test-watch [TEST=path]' \
-		'  make test-hooks-agents [TEST=tests_hooks_agents/...] [WORKERS=auto] [ARGS="-v"]' \
-		'  make test-relay-plugin [TEST=tests_relay_plugin/...] [ARGS="-v"]' \
+		'  make test-hooks-agents [TEST=integration_tests/hooks_agents/...] [WORKERS=auto] [ARGS="-v"]' \
+		'  make test-relay-plugin [TEST=integration_tests/relay_plugin/...] [ARGS="-v"]' \
 		'  make record-cassettes [RECORDED_TESTS=tests/recorded] [RECORDED_RECORD_MODE=once] [RECORDED_SNAPSHOT_MODE=create] [RECORDED_REQUIRED_KEYS="OPENAI_API_KEY NVIDIA_API_KEY"]' \
 		'  make rewrite-cassettes [RECORDED_TESTS=tests/recorded] [RECORDED_REQUIRED_KEYS="OPENAI_API_KEY NVIDIA_API_KEY"]' \
 		'  make replay-cassettes [RECORDED_TESTS=tests/recorded]' \

crates/nemo-guardrails-relay-plugin/README.md

@@ -71,15 +71,16 @@ make test TEST=crates/nemo-guardrails-relay-plugin/scripts/test_package_bundle.p
 None of these tests needs a running NeMo Guardrails service.
 `.cargo/config.toml` sets `NEMO_RELAY_TEST_SKIP_IMPLICIT_CONFIG=1`, so that the host tests ignore a machine-wide `/etc/nemo-relay/plugins.toml`.
 
-The end-to-end tests in `tests_relay_plugin/`, at the repository root, load the release library into Relay's Python plugin host and check calls against a real NeMo Guardrails service: managed tool calls, and a LangChain agent through Relay's middleware.
-They start the service from the repository's `.venv`, as a hooks server for each of their configurations, and run in their own Python 3.11 environment without Guardrails, `.venv-relay-plugin`, installed from the hash-pinned `tests_relay_plugin/requirements.txt`.
+The end-to-end tests in the repository's `integration_tests/relay_plugin/` load the release library into Relay's Python plugin host and check calls against a real NeMo Guardrails service: managed tool calls, and a LangChain agent through Relay's middleware.
+They start the service from the repository's `.venv`, as a hooks server for each of their configurations, and run in their own Python 3.11 environment without Guardrails, `.venv-relay-plugin`, installed from the hash-pinned `integration_tests/relay_plugin/requirements.txt`.
 They run on Linux and macOS, need the Rust toolchain, and aren't part of `make test`.
+[`integration_tests/README.md`](../../integration_tests/README.md) covers all of the repository's end-to-end suites, this one included.
 From the repository root:
 
 ```sh
 make test-relay-plugin
 # One file or test, with pytest's options
-make test-relay-plugin TEST=tests_relay_plugin/test_langchain.py ARGS=-v
+make test-relay-plugin TEST=integration_tests/relay_plugin/test_langchain.py ARGS=-v
 ```
 
 ## Packaging

integration_tests/README.md

@@ -0,0 +1,70 @@
+# Integration Tests
+
+The suites here run NeMo Guardrails against the real agents and clients that call it, with fake models behind them, so no test calls a live model or needs an API key.
+Each suite installs those programs from its own hash-pinned `requirements.txt`, so none of them is a Guardrails dependency, and each has its own Make target.
+
+`make test` doesn't run these suites, since each needs programs from outside Guardrails' environment: they sit outside the `testpaths` in `pytest.ini`.
+They also sit outside `tests/`, so they don't inherit the main suite's fixtures from `tests/conftest.py`.
+The hooks suite runs in `.venv` and the Relay suite in `.venv-relay-plugin`, so run each through its own target: `pytest integration_tests` fails at collection.
+
+| Suite | What it checks | Command |
+| --- | --- | --- |
+| [`hooks_agents/`](hooks_agents/) | Claude Code and Codex obey the hooks server's answers | `make test-hooks-agents` |
+| [`relay_plugin/`](relay_plugin/) | The NeMo Relay plugin checks tool calls with a real hooks server | `make test-relay-plugin` |
+
+## Claude Code and Codex (`hooks_agents/`)
+
+The suite checks that Claude Code and Codex obey the hooks server through the `PreToolUse` hooks their docs configure, with the configs in `examples/configs/agent_hooks`.
+A call the config allows runs, and a call it denies never runs, with the agent telling its model the server's reason.
+A call also never runs when its rail fails, when the hooks server is down, or when the server doesn't answer within the hook's timeout.
+
+Each test runs the agent headless against a fake model on loopback, which calls one scripted shell command.
+The hooks server serves the test's config in the test's own process, except in the tests of a server that's down or doesn't answer: their hook points at a loopback port that refuses the connection, or at one that accepts it and never answers.
+The agents are the real binaries from their published wheels.
+
+Prerequisites:
+
+- Linux or macOS, since the docs' hooks and the scripted command need a POSIX shell.
+- `curl` on `PATH`, which Codex's hook from the docs runs to call the hooks server.
+- Network access to PyPI the first time, to download the agents' wheels.
+- On a managed machine, Claude Code's managed settings still apply: an `allowedHttpHookUrls` that doesn't allow `localhost` stops its hook from running.
+
+From the repository root:
+
+```sh
+make test-hooks-agents
+# One file or test, with pytest's options
+make test-hooks-agents TEST=integration_tests/hooks_agents/test_hooks_agents.py::test_allowed_call_runs ARGS=-v
+```
+
+`make test-hooks-agents` installs `claude-agent-sdk`, which bundles Claude Code, and `openai-codex-cli-bin`, which bundles Codex, into `.cache/hooks_agents`, checking each wheel against its hash in `requirements.txt`.
+It then runs the suite in `.venv`, with live provider keys unset as for `make test`.
+To change an agent's version, edit its pin in `requirements.in`, then recompile `requirements.txt` with the command that `requirements.in` gives.
+
+## NeMo Relay Plugin (`relay_plugin/`)
+
+The suite builds the plugin's release library with Cargo, loads it into Relay's Python plugin host, and checks tool calls against a real NeMo Guardrails hooks server: calls that Relay manages, and a LangChain agent through Relay's middleware.
+A call the hooks server allows runs, and a call that the tool policy or a rail denies never runs, with the plugin rejecting it with the server's reason.
+A call also never runs when the judge model gives no verdict, when the check outlasts the plugin's `timeout_millis`, or when the hooks server can't be reached.
+A scripted chat model drives the LangChain agent, and a stand-in judge answers the configurations that use a judge model.
+
+The suite runs in its own Python 3.11 environment, `.venv-relay-plugin`, without Guardrails: `nemo-relay` needs a newer `langchain` than `uv.lock` pins.
+The hooks servers run from the repository's `.venv`, one for each configuration.
+
+Prerequisites:
+
+- Linux or macOS, since the suite loads the plugin as a `.so` or `.dylib`.
+- The Rust toolchain from rustup, which reads `rust-toolchain.toml`, to build the plugin's release library.
+- No machine-wide `/etc/nemo-relay/plugins.toml`: Relay would merge it into every test's plugin host, so the suite fails when it exists.
+- Network access the first time, to download the packages in `requirements.txt`, the Rust toolchain that `rust-toolchain.toml` pins, the plugin's Rust crates and, when the machine lacks it, Python 3.11.
+
+From the repository root:
+
+```sh
+make test-relay-plugin
+# One file or test, with pytest's options
+make test-relay-plugin TEST=integration_tests/relay_plugin/test_langchain.py ARGS=-v
+```
+
+`make test-relay-plugin` syncs `.venv`, creates `.venv-relay-plugin` with Python 3.11, and installs it from `requirements.txt`, checking every package against its hash.
+To change the `nemo-relay` version, edit its pin in `requirements.in`, then recompile `requirements.txt` with the command that `requirements.in` gives.

integration_tests/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests_hooks_agents/agents.py → integration_tests/hooks_agents/agents.py

@@ -25,7 +25,7 @@
 
 import pytest
 
-from tests_hooks_agents.fake_model import FakeModel, ScriptedToolCall
+from integration_tests.hooks_agents.fake_model import FakeModel, ScriptedToolCall
 
 # The scripted shell command writes this file in the agent's working directory, so a test can tell whether it ran.
 TOOL_RAN_FILE = "tool-ran.txt"

tests_hooks_agents/conftest.py → integration_tests/hooks_agents/conftest.py

@@ -37,11 +37,11 @@
 import pytest
 import uvicorn
 
+from integration_tests.hooks_agents.agents import Agent, ClaudeCode, Codex
 from nemoguardrails import RailsConfig
 from nemoguardrails.server.hooks.app import create_hooks_app
-from tests_hooks_agents.agents import Agent, ClaudeCode, Codex
 
-REPO_ROOT = Path(__file__).resolve().parents[1]
+REPO_ROOT = Path(__file__).resolve().parents[2]
 # Where `make test-hooks-agents` installs the agents' wheels: HOOKS_AGENTS_DIR when it's set, against the repository root.
 AGENTS_DIR = REPO_ROOT / (os.environ.get("HOOKS_AGENTS_DIR") or ".cache/hooks_agents")
 

tests_hooks_agents/requirements.in → integration_tests/hooks_agents/requirements.in

@@ -4,7 +4,7 @@
 # binaries, and with hashes, which `make test-hooks-agents` checks as it installs them. Compile it from the repository
 # root, which applies the repository's 7-day exclude-newer window:
 #
-#   uv pip compile --universal --no-deps --generate-hashes tests_hooks_agents/requirements.in \
-#     -o tests_hooks_agents/requirements.txt
+#   uv pip compile --universal --no-deps --generate-hashes integration_tests/hooks_agents/requirements.in \
+#     -o integration_tests/hooks_agents/requirements.txt
 claude-agent-sdk==0.2.168
 openai-codex-cli-bin==0.163.0

tests_hooks_agents/requirements.txt → integration_tests/hooks_agents/requirements.txt

@@ -1,13 +1,13 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile --universal --no-deps --generate-hashes tests_hooks_agents/requirements.in -o tests_hooks_agents/requirements.txt
+#    uv pip compile --universal --no-deps --generate-hashes integration_tests/hooks_agents/requirements.in -o integration_tests/hooks_agents/requirements.txt
 claude-agent-sdk==0.2.168 \
     --hash=sha256:2ef1e7fc442c5abdc1e698027bf1b5faa8ea2130134a14364e2ae3c53bf31fdd \
     --hash=sha256:653fc7bbe56630e6ddd9601a7cfe50586667e74102da3fb90571456c63655a62 \
     --hash=sha256:77ae66b10edfd07b6ea3dec2af106ad47a9d616f8eb52d4f9b385e1b00bb2a27 \
     --hash=sha256:80388054a09c137d76fc29caa6e46bc90d2ae2ef09fac5ee1c38fc36d2762fda \
     --hash=sha256:be197be92549909ec62a2cee04c27f4b0a38c80f723735538a3efc352507f81d \
     --hash=sha256:ed1814b0dd1c113bdaf3e51e6a67aa5b96b8f5c5839a43aeb6ae1e6ed23a5b35
-    # via -r tests_hooks_agents/requirements.in
+    # via -r integration_tests/hooks_agents/requirements.in
 openai-codex-cli-bin==0.163.0 \
     --hash=sha256:0e2984865a16661eddf431ff26d5effd24fa00175b9e8b7a7e2c889276707cca \
     --hash=sha256:64c7d113e1bf6cefa87159c1d21115654a716e1b505295d8520aa91e399ad115 \
@@ -17,4 +17,4 @@ openai-codex-cli-bin==0.163.0 \
     --hash=sha256:b759f212cc0deec6da9f297800797232c73d3f0d963842deaa02df965ee0b8b6 \
     --hash=sha256:bb2b0d117eeb02608738cf2bcdf51f6a036343d0028272e11a94e72b0d23b141 \
     --hash=sha256:dd0e6570930167f0e15ec94c5fb494a4a5a9dceb0a97113176c7b47c826cda05
-    # via -r tests_hooks_agents/requirements.in
+    # via -r integration_tests/hooks_agents/requirements.in

tests_hooks_agents/test_hooks_agents.py → integration_tests/hooks_agents/test_hooks_agents.py

@@ -19,12 +19,12 @@
 
 import pytest
 
+from integration_tests.hooks_agents.agents import TOOL_RAN_FILE, Agent
 from nemoguardrails import RailsConfig
-from tests_hooks_agents.agents import TOOL_RAN_FILE, Agent
 
 pytestmark = pytest.mark.slow
 
-EXAMPLE_CONFIGS = Path(__file__).resolve().parents[1] / "examples" / "configs" / "agent_hooks"
+EXAMPLE_CONFIGS = Path(__file__).resolve().parents[2] / "examples" / "configs" / "agent_hooks"
 
 ALLOWED_COMMAND = f"echo hello > {TOOL_RAN_FILE}"
 # The example configs' Bash rail matches `rm -rf`, which only echoing it keeps harmless, if the agent runs it anyway.

tests_relay_plugin/conftest.py → integration_tests/relay_plugin/conftest.py

@@ -35,7 +35,7 @@
 from mock_judge import MockJudge
 from nemo_relay import plugin
 
-REPO_ROOT = Path(__file__).resolve().parents[1]
+REPO_ROOT = Path(__file__).resolve().parents[2]
 CRATE_DIR = REPO_ROOT / "crates" / "nemo-guardrails-relay-plugin"
 LIBRARY_TARGET = "nemo_guardrails_relay_plugin"
 LIBRARY_SUFFIX = ".dylib" if sys.platform == "darwin" else ".so"

integration_tests/relay_plugin/pytest.ini

@@ -0,0 +1,3 @@
+# The end-to-end suite's own pytest config, so that `pytest integration_tests/relay_plugin` doesn't read the
+# repository's pytest.ini, which is for Guardrails' test environment.
+[pytest]

tests_relay_plugin/requirements.in → integration_tests/relay_plugin/requirements.in

@@ -5,6 +5,6 @@
 # pinned exactly here; `make test-relay-plugin` exempts it the same way:
 #
 #   uv pip compile --universal --python-version 3.11 --generate-hashes --exclude-newer-package nemo-relay=false \
-#     tests_relay_plugin/requirements.in -o tests_relay_plugin/requirements.txt
+#     integration_tests/relay_plugin/requirements.in -o integration_tests/relay_plugin/requirements.txt
 nemo-relay[langchain]==0.10.4
 pytest