feat: integration tests on CI

.claude/rules/integration-testing.md

@@ -12,11 +12,27 @@ paths:
 ## Running
 
 ```bash
-make integration_test MODEL=<model>                  # from cache
-REFRESH=TRUE make integration_test MODEL=<model>     # build cache against live DIAL Core(only if it is absence)
+make integration_test MODEL=<model>                  # from cache (single model)
+make integration_test_all                            # all models in INTEGRATION_TEST_MODELS (sequential, local)
+make integration_test_ci MODEL=<model>               # CI entry point (single model, fails if MODEL unset)
+REFRESH=TRUE make integration_test MODEL=<model>     # build cache against live DIAL Core (only if absent)
+REFRESH=TRUE make integration_test_all               # refresh cache for all configured models
 make e2e_test                                         # happy-path e2e (always live)
 ```
 
+### CI (optional, manual)
+
+Integration tests are **not** part of the PR gate. To run the full model matrix on CI:
+
+1. GitHub → **Actions** → **Integration Tests** → **Run workflow**
+2. Or: `gh workflow run integration-tests.yml --ref <branch>`
+
+The workflow runs **3 parallel jobs** (one per model in `INTEGRATION_TEST_MODELS` in the `Makefile`).
+
+### Model matrix configuration
+
+Edit `INTEGRATION_TEST_MODELS` at the top of the `Makefile` integration-test section (defaults: gemini, gpt, claude). When deployments change, update that list and refresh cache as needed.
+
 Prerequisites in `.env`:
 ```
 REMOTE_DIAL_URL=...

.github/workflows/integration-tests.yml

@@ -0,0 +1,63 @@
+name: Integration Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Git branch, tag, or SHA to test
+        required: false
+        default: ""
+
+concurrency:
+  group: integration-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    runs-on: ubuntu-24.04
+    outputs:
+      models: ${{ steps.matrix.outputs.models }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+          lfs: true
+      - id: matrix
+        run: |
+          models=$(make -s print-integration-test-models | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "models=$models" >> "$GITHUB_OUTPUT"
+
+  integration_test:
+    name: integration (${{ matrix.model }})
+    needs: prepare
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        model: ${{ fromJson(needs.prepare.outputs.models) }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+          lfs: true
+      - uses: epam/ai-dial-ci/actions/python_prepare@4.7.0
+        with:
+          python-version: "3.13"
+          poetry-version: "2.3.2"
+      - name: Integration tests
+        env:
+          DIAL_URL: ${{ secrets.DIAL_URL }}
+          DIAL_API_KEY: ${{ secrets.DIAL_API_KEY }}
+          PY_INTERPRETER_URL: ${{ secrets.PY_INTERPRETER_URL }}
+          PY_INTERPRETER_API_KEY: ${{ secrets.PY_INTERPRETER_API_KEY }}
+          PY_INTERPRETER_LOCAL_RUN: "true"
+        run: make integration_test_ci MODEL=${{ matrix.model }}
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: ${{ !cancelled() }}
+        with:
+          name: integration-junit-${{ matrix.model }}
+          path: reports/tests-integration-*.xml
+          retention-days: 30

CLAUDE.md

@@ -27,7 +27,9 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 | Unit tests | `make test` | |
 | Unit tests (filtered) | `make test ARGS="-k test_name -x"` | |
 | Unit tests + coverage | `make test_cov` | |
-| Integration tests | `make integration_test MODEL=<model>` | Automatically starts/stops the local MCP + REST test server |
+| Integration tests (single model) | `make integration_test MODEL=<model>` | Automatically starts/stops the local MCP + REST test server |
+| Integration tests (all models, local) | `make integration_test_all` | Sequential run over `INTEGRATION_TEST_MODELS` in Makefile |
+| Integration tests (CI) | `make integration_test_ci MODEL=<model>` | Used by the manual **Integration Tests** GitHub Actions workflow |
 | E2E tests | `make e2e_test` | |
 | Dump app schema | `make dump_app_schema` | |
 

Makefile

@@ -3,6 +3,7 @@ MYPY_DIRS = src/quickapp src/scripts
 FILES ?= $(SRC_DIRS)
 POETRY ?= poetry
 PYTHON ?= python3
+WORKERS ?= 4
 
 -include .env
 export
@@ -13,7 +14,8 @@ export PYDANTIC_V2=True
 .PHONY: init_venv install install_dev install_integration install_all clean \
 	lint mypy format install_pre_commit_hooks run_chat run_error_injection_app test test_cov \
 	dump_app_schema dump_internal_tools dump_config_support_openapi generate_dial_config start_test_server stop_test_server \
-	integration_test integration_test_run e2e_test run_python \
+	integration_test integration_test_run integration_test_ci integration_test_all print-integration-test-models \
+	e2e_test run_python \
 	black black_check isort isort_check autoflake autoflake_check flake8
 
 init_venv:
@@ -129,35 +131,63 @@ generate_dial_config: install_dev
 	--config docker_compose_files/core/configuration/generated/models.json \
 	--applications dial-rag,dial-web-rag
 
-start_test_server:
+start_test_server: stop_test_server
 	echo "Starting MCP + REST servers..."
 	$(POETRY) run python src/tests/integration_tests/data_server_for_tests.py & echo $$! > .mcp_rest_server.pid
 	sleep 1
 	echo "Servers started with PID `cat .mcp_rest_server.pid`"
 
 stop_test_server:
-	@if [ -f .mcp_rest_server.pid ]; then \
-		pid=$$(cat .mcp_rest_server.pid); \
-		if kill -0 $$pid >/dev/null 2>&1; then \
-			echo "Stopping MCP + REST servers..."; \
-			kill $$pid; \
-			rm -f .mcp_rest_server.pid; \
-			echo "Servers stopped"; \
-		else \
-			echo "No running process found with PID $$pid"; \
-			rm -f .mcp_rest_server.pid; \
-		fi \
-	else \
-		echo "PID file not found. Are servers running?"; \
-	fi
+	@$(POETRY) run python src/tests/integration_tests/stop_data_server_for_tests.py stop || true
+
+# Canonical integration-test matrix (one per provider family). Update when DIAL deployments change.
+INTEGRATION_TEST_MODELS ?= \
+	gemini-3.5-flash \
+	gpt-5.5-2026-04-24 \
+	anthropic.claude-opus-4-6-v1
+
+# Filesystem-safe JUnit report suffix (Claude deployment IDs may contain ':').
+MODEL_REPORT_SUFFIX = $(subst /,-,$(subst :,-,$(MODEL)))
+
+INTEGRATION_PYTEST = ENABLE_PREVIEW_FEATURES=true $(POETRY) run pytest -n $(or ${WORKERS},logical) \
+	src/tests/integration_tests --model=$(MODEL) \
+	--junitxml=reports/tests-integration-$(MODEL_REPORT_SUFFIX).xml \
+	-m "integration" $(ARGS)
+
+print-integration-test-models:
+	@printf '%s\n' $(INTEGRATION_TEST_MODELS)
 
 integration_test: install_integration
 	$(MAKE) start_test_server
-	ENABLE_PREVIEW_FEATURES=true $(POETRY) run pytest -n $(or ${WORKERS},logical) src/tests/integration_tests --model=${MODEL} --junitxml=reports/tests-integration-${MODEL_SHORT_NAME}.xml -m "integration" $(ARGS)
-	$(MAKE) stop_test_server
+	@status=0; \
+	$(INTEGRATION_PYTEST) || status=$$?; \
+	$(MAKE) stop_test_server; \
+	exit $$status
 
 integration_test_run:
-	ENABLE_PREVIEW_FEATURES=true $(POETRY) run pytest --model=${MODEL} --junitxml=reports/tests-integration-${MODEL_SHORT_NAME}.xml -m "integration" $(ARGS)
+	$(INTEGRATION_PYTEST)
+
+integration_test_ci:
+ifndef MODEL
+	$(error MODEL is required, e.g. make integration_test_ci MODEL=gemini-2.5-pro)
+endif
+	$(MAKE) integration_test MODEL=$(MODEL)
+
+integration_test_all: install_integration
+	$(MAKE) start_test_server
+	@failed=""; \
+	for model in $(INTEGRATION_TEST_MODELS); do \
+		echo "=== Integration tests: $$model ==="; \
+		if ! $(MAKE) integration_test_run MODEL="$$model"; then \
+			echo "=== FAILED: $$model ==="; \
+			failed="$$failed $$model"; \
+		fi; \
+	done; \
+	$(MAKE) stop_test_server; \
+	if [ -n "$$failed" ]; then \
+		echo "Integration tests failed for:$$failed"; \
+		exit 1; \
+	fi
 
 e2e_test: install_integration
 	ENABLE_PREVIEW_FEATURES=true $(POETRY) run pytest -n $(or ${WORKERS},logical) --no-cache --junitxml=reports/tests-e2e.xml -m "e2e" $(ARGS)

src/tests/integration_tests/README.md

@@ -39,7 +39,17 @@ Notes:
 
 ## 3. Execute tests
 - Run end-to-end tests: `make e2e_test`
-- Run integration tests: `make integration_test`
+- Run integration tests (single model): `make integration_test MODEL=<model>`
+- Run integration tests (all configured models, local): `make integration_test_all`
+
+### CI (optional, manual)
+
+Integration tests are not run on every PR. To run the full model matrix on GitHub Actions:
+
+- **Actions** → **Integration Tests** → **Run workflow**
+- Or: `gh workflow run integration-tests.yml --ref <branch>`
+
+Models are configured in `INTEGRATION_TEST_MODELS` in the root `Makefile`. CI runs one parallel job per model via `make integration_test_ci MODEL=<model>`.
 
 ## Test types (brief)
 - `e2e`:

src/tests/integration_tests/stop_data_server_for_tests.py

@@ -0,0 +1,126 @@
+"""Helpers to start/stop the MCP + REST dual server used by integration tests."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+# Ports must stay in sync with DualServerRunner defaults in data_server_for_tests.py
+REST_PORT = 8002
+MCP_PORT = 8003
+PORTS = (REST_PORT, MCP_PORT)
+PID_FILE = Path(".mcp_rest_server.pid")
+
+
+def _pids_listening_on_port(port: int) -> set[int]:
+    """Return PIDs that currently listen on ``port`` (best-effort, OS-specific)."""
+    pids: set[int] = set()
+    if sys.platform == "win32":
+        result = subprocess.run(
+            ["netstat", "-ano", "-p", "tcp"],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        for line in result.stdout.splitlines():
+            # Example: TCP    0.0.0.0:8002    0.0.0.0:0    LISTENING    12345
+            parts = line.split()
+            if len(parts) < 5 or parts[0].upper() != "TCP":
+                continue
+            local_addr = parts[1]
+            state = parts[3].upper() if len(parts) >= 5 else ""
+            if state != "LISTENING":
+                continue
+            if not local_addr.endswith(f":{port}"):
+                continue
+            try:
+                pids.add(int(parts[-1]))
+            except ValueError:
+                continue
+    else:
+        result = subprocess.run(
+            ["lsof", "-ti", f"TCP:{port}", "-sTCP:LISTEN"],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        for token in result.stdout.split():
+            try:
+                pids.add(int(token))
+            except ValueError:
+                continue
+    return pids
+
+
+def _kill_pid(pid: int) -> None:
+    if pid <= 0:
+        return
+    if sys.platform == "win32":
+        subprocess.run(
+            ["taskkill", "/F", "/T", "/PID", str(pid)],
+            capture_output=True,
+            check=False,
+        )
+    else:
+        try:
+            os.kill(pid, signal.SIGTERM)
+        except ProcessLookupError:
+            return
+        for _ in range(20):
+            try:
+                os.kill(pid, 0)
+            except ProcessLookupError:
+                return
+            time.sleep(0.05)
+        try:
+            os.kill(pid, signal.SIGKILL)
+        except ProcessLookupError:
+            return
+
+
+def stop_servers() -> None:
+    """Stop the dual server by PID file and by freeing known listen ports."""
+    pids: set[int] = set()
+    if PID_FILE.exists():
+        try:
+            pids.add(int(PID_FILE.read_text(encoding="utf-8").strip()))
+        except ValueError:
+            pass
+        PID_FILE.unlink(missing_ok=True)
+
+    for port in PORTS:
+        pids.update(_pids_listening_on_port(port))
+
+    for pid in sorted(pids):
+        print(f"Stopping MCP/REST test server process {pid}...")
+        _kill_pid(pid)
+
+    # Brief wait so TIME_WAIT / WinError 10048 does not race the next bind.
+    time.sleep(0.5)
+    still_busy = {port: _pids_listening_on_port(port) for port in PORTS}
+    busy = {port: p for port, p in still_busy.items() if p}
+    if busy:
+        print(f"Warning: ports still in use after stop: {busy}", file=sys.stderr)
+        sys.exit(1)
+    print("MCP + REST test servers stopped")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "action",
+        choices=("stop",),
+        help="stop: kill PID file process and free ports 8002/8003",
+    )
+    args = parser.parse_args()
+    if args.action == "stop":
+        stop_servers()
+
+
+if __name__ == "__main__":
+    main()

src/tests/integration_tests/test_orchestrator_contexts.py

@@ -13,7 +13,7 @@
 
 # Deployments that accept PDF input attachments well enough for this scenario (extend as needed).
 _LAZY_CONTEXT_MODELS = [
-    "gpt-5.2-2025-12-11",
+    "gpt-5.5-2026-04-24",
     "gpt-5-2025-08-07",
     "anthropic.claude-opus-4-6-v1",
 ]
@@ -23,7 +23,6 @@
 @e2e_test(
     config_file_set="lazy_admin_context",
     models_applicable_for_test=_LAZY_CONTEXT_MODELS,
-    runs=1,
     include_rest_toolset=False,
     application_context_files=[
         _DOCS / "ontologies.pdf",
@@ -35,15 +34,6 @@
         "Two admin PDFs; model must list then get_content the ontology doc only",
         similarity_threshold=0.8,
     )
-    .add_user_message(
-        user_message="enlist all tools that you have",
-        answer=[
-            "I have access to internal_attachments_available_context and "
-            "internal_attachments_get_content tools for listing and loading admin context files.",
-            "Available tools: internal_attachments_available_context (lists admin files) and "
-            "internal_attachments_get_content (loads a specific file).",
-        ],
-    )
     .add_user_message(
         user_message=(
             "Admin context includes two PDF files: one about ontology tooling and one IMF WEO. "
@@ -55,7 +45,7 @@
         ),
         tool_calls=[
             ToolCall(
-                ToolNames.INTERNAL_ATTACHMENTS_AVAILABLE_CONTEXT.value, min_calls=1, max_calls=4
+                ToolNames.INTERNAL_ATTACHMENTS_AVAILABLE_CONTEXT.value, min_calls=0, max_calls=4
             ),
             ToolCall(ToolNames.INTERNAL_ATTACHMENTS_GET_CONTENT.value, min_calls=1, max_calls=4),
         ],