diff --git a/airbyte/mcp/__init__.py b/airbyte/mcp/__init__.py index 0361e43fd..1a718274a 100644 --- a/airbyte/mcp/__init__.py +++ b/airbyte/mcp/__init__.py @@ -19,6 +19,29 @@ 2. Register the MCP server with your MCP client. 3. Test the MCP server connection using your MCP client. +### Hosted HTTPS deployment (remote MCP clients) + +Remote MCP clients require a public **HTTPS** endpoint. They cannot use the local +stdio transport. + +Use the **`airbyte-mcp-http`** entry point instead, which serves Streamable HTTP on port +8080. Terminate TLS at a reverse proxy or load balancer in front of the server. + +```bash +# Docker (production transport) +docker build -f Dockerfile.mcp -t airbyte-mcp . +docker run --rm -p 8080:8080 \ + -e AIRBYTE_MCP_ENV_FILE=/secrets/airbyte_mcp.env \ + -v "/path/to/airbyte_mcp.env:/secrets/airbyte_mcp.env:ro" \ + airbyte-mcp + +# Local dev with the same transport as production +poe mcp-serve-streamable-http +``` + +For a full deployment guide (HTTPS, remote client integration, OIDC, CORS), see +[docs/MCP_HTTP_DEPLOYMENT.md](https://github.com/airbytehq/PyAirbyte/blob/main/docs/MCP_HTTP_DEPLOYMENT.md). + ### Step 1: Generate a Dotenv Secrets File To get started with the Airbyte Replication MCP server, you will need to create a dotenv diff --git a/airbyte/mcp/_http_config.py b/airbyte/mcp/_http_config.py new file mode 100644 index 000000000..1933ccb7e --- /dev/null +++ b/airbyte/mcp/_http_config.py @@ -0,0 +1,57 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""HTTP transport configuration helpers for hosted MCP deployment.""" + +from __future__ import annotations + +import os + +from starlette.middleware import Middleware +from starlette.middleware.cors import CORSMiddleware + + +MCP_CORS_ORIGINS_ENV = "MCP_CORS_ORIGINS" + +# Headers used by remote MCP clients over streamable HTTP. +_MCP_CLIENT_HEADERS = [ + "mcp-protocol-version", + "mcp-session-id", + "Authorization", + "Content-Type", + "X-Airbyte-Workspace-Id", + "X-Airbyte-Cloud-Client-Id", + "X-Airbyte-Cloud-Client-Secret", + "X-Airbyte-Cloud-Api-Url", + "X-Airbyte-Cloud-Config-Api-Url", +] + + +def parse_cors_origins(raw: str | None) -> list[str]: + """Parse a comma-separated list of CORS origins.""" + if not raw: + return [] + return [origin.strip() for origin in raw.split(",") if origin.strip()] + + +def build_http_middleware( + *, + cors_origins: str | None = None, +) -> list[Middleware]: + """Build ASGI middleware for the hosted MCP HTTP transport. + + When ``MCP_CORS_ORIGINS`` is set (comma-separated), enables CORS for + remote MCP clients that connect from a browser context (for example, + browser-based OAuth flows). + """ + origins = parse_cors_origins(cors_origins or os.getenv(MCP_CORS_ORIGINS_ENV, "")) + if not origins: + return [] + + return [ + Middleware( + CORSMiddleware, + allow_origins=origins, + allow_methods=["GET", "POST", "DELETE", "OPTIONS"], + allow_headers=_MCP_CLIENT_HEADERS, + expose_headers=["mcp-protocol-version", "mcp-session-id"], + ), + ] diff --git a/airbyte/mcp/http_main.py b/airbyte/mcp/http_main.py index 226198ba1..1b39e1ac0 100644 --- a/airbyte/mcp/http_main.py +++ b/airbyte/mcp/http_main.py @@ -10,9 +10,13 @@ - `MCP_SERVER_URL`: Public base URL. Used for OIDC redirect callbacks and to derive the MCP endpoint mount path (serves at `/` when the URL has a path prefix, otherwise defaults to `/mcp`). +- `MCP_CORS_ORIGINS`: Comma-separated CORS origins for remote MCP clients + (for example, browser-based OAuth flows). - `OIDC_CONFIG_URL`: Keycloak OIDC discovery URL (enables auth with all three OIDC vars) - `OIDC_CLIENT_ID`: OIDC client identifier - `OIDC_CLIENT_SECRET`: OIDC client secret + +See `docs/MCP_HTTP_DEPLOYMENT.md` for HTTPS deployment and remote client integration. """ from __future__ import annotations @@ -21,6 +25,7 @@ import os from urllib.parse import urlparse +from airbyte.mcp._http_config import build_http_middleware from airbyte.mcp.server import ( DEFAULT_HTTP_HOST, DEFAULT_HTTP_PORT, @@ -52,12 +57,15 @@ def main() -> None: mcp_path, ) + middleware = build_http_middleware() + app.run( transport="streamable-http", host=DEFAULT_HTTP_HOST, port=DEFAULT_HTTP_PORT, path=mcp_path, stateless_http=True, + middleware=middleware or None, ) diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 000000000..d7abd8cc2 --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,23 @@ +# Caddy reverse proxy for the Airbyte Replication MCP server. +# +# Usage (with docker-compose.mcp.yml): +# MCP_PUBLIC_HOST=mcp.example.com docker compose -f deploy/docker-compose.mcp.yml up -d +# +# Caddy obtains and renews TLS certificates automatically for MCP_PUBLIC_HOST. + +{$MCP_PUBLIC_HOST} { + # Health checks for load balancers and uptime monitors. + handle /health { + reverse_proxy mcp:8080 + } + + # Streamable HTTP MCP endpoint (default mount path is /mcp). + handle /mcp* { + reverse_proxy mcp:8080 + } + + # Fallback for path-stripped deployments (MCP_SERVER_URL with a path prefix). + handle { + reverse_proxy mcp:8080 + } +} diff --git a/deploy/docker-compose.mcp.yml b/deploy/docker-compose.mcp.yml new file mode 100644 index 000000000..056cb44f7 --- /dev/null +++ b/deploy/docker-compose.mcp.yml @@ -0,0 +1,57 @@ +# Self-hosted Airbyte Replication MCP server with automatic HTTPS via Caddy. +# +# Prerequisites: +# - DNS for MCP_PUBLIC_HOST must point to this host +# - A dotenv secrets file at ~/.mcp/airbyte_mcp.env (or override MCP_ENV_FILE) +# +# Usage: +# export MCP_PUBLIC_HOST=mcp.example.com +# docker compose -f deploy/docker-compose.mcp.yml up -d --build +# +# Public MCP endpoint (default mount): https://$MCP_PUBLIC_HOST/mcp + +services: + mcp: + build: + context: .. + dockerfile: Dockerfile.mcp + restart: unless-stopped + environment: + AIRBYTE_MCP_ENV_FILE: /secrets/airbyte_mcp.env + MCP_SERVER_URL: https://${MCP_PUBLIC_HOST:-localhost} + MCP_CORS_ORIGINS: ${MCP_CORS_ORIGINS:-} + OIDC_CONFIG_URL: ${OIDC_CONFIG_URL:-} + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-} + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-} + AIRBYTE_CLOUD_MCP_SAFE_MODE: ${AIRBYTE_CLOUD_MCP_SAFE_MODE:-1} + AIRBYTE_CLOUD_MCP_READONLY_MODE: ${AIRBYTE_CLOUD_MCP_READONLY_MODE:-0} + volumes: + - ${MCP_ENV_FILE:-${HOME}/.mcp/airbyte_mcp.env}:/secrets/airbyte_mcp.env:ro + expose: + - "8080" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + + caddy: + image: caddy:2-alpine + restart: unless-stopped + ports: + - "80:80" + - "443:443" + environment: + MCP_PUBLIC_HOST: ${MCP_PUBLIC_HOST:-localhost} + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + depends_on: + mcp: + condition: service_healthy + +volumes: + caddy_data: + caddy_config: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 8a5778134..4e4a0d4e0 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -137,13 +137,17 @@ poe mcp-tool-test list_deployed_cloud_connections '{}' You can also invoke the server using one of these helper tasks: ```bash -poe mcp-serve-local # STDIO transport (default) -poe mcp-serve-http # HTTP transport on localhost:8000 -poe mcp-serve-sse # Server-Sent Events transport on localhost:8000 +poe mcp-serve-local # STDIO transport (default) +poe mcp-serve-http # HTTP transport on localhost:8000 +poe mcp-serve-sse # Server-Sent Events transport on localhost:8000 +poe mcp-serve-streamable-http # Streamable HTTP on localhost:8080 (production transport) poe mcp-inspect # Show all available MCP tools and their schemas ``` +For hosted HTTPS deployment (remote MCP clients), see +[docs/MCP_HTTP_DEPLOYMENT.md](./MCP_HTTP_DEPLOYMENT.md). + ### Generating Markdown docs for the MCP Server The repo ships a small script (`scripts/generate_mcp_markdown.py`) that diff --git a/docs/MCP_HTTP_DEPLOYMENT.md b/docs/MCP_HTTP_DEPLOYMENT.md new file mode 100644 index 000000000..feb99e766 --- /dev/null +++ b/docs/MCP_HTTP_DEPLOYMENT.md @@ -0,0 +1,172 @@ +# Hosting the Airbyte Replication MCP Server over HTTPS + +The Airbyte Replication MCP server supports remote clients that require a public HTTPS endpoint. The local stdio transport (`airbyte-mcp`) is for desktop MCP clients only. + +For hosted deployment, use the **`airbyte-mcp-http`** entry point, which serves the MCP protocol over **Streamable HTTP** — the transport remote MCP clients expect. + +## Architecture + +```text +Remote MCP client (MCP Inspector, hosted agent platform, etc.) + │ HTTPS + ▼ +Reverse proxy / load balancer (TLS termination) + │ HTTP :8080 + ▼ +airbyte-mcp-http (streamable-http, stateless) + │ + ▼ +Airbyte Cloud / local connector tools +``` + +TLS is terminated at the reverse proxy or load balancer. The MCP server itself listens on plain HTTP (`0.0.0.0:8080`). + +There is **no official Airbyte-hosted public MCP endpoint** in this repository. You self-host the server in your own infrastructure and expose it at a public HTTPS URL. + +## Quick start with Docker + +Build and run the MCP server container: + +```bash +docker build -f Dockerfile.mcp -t airbyte-mcp . +docker run --rm -p 8080:8080 \ + -e AIRBYTE_MCP_ENV_FILE=/secrets/airbyte_mcp.env \ + -v "$HOME/.mcp/airbyte_mcp.env:/secrets/airbyte_mcp.env:ro" \ + airbyte-mcp +``` + +For HTTPS with automatic certificates, use the included Compose stack: + +```bash +# Set your public hostname, then start Caddy + MCP server +export MCP_PUBLIC_HOST=mcp.example.com +docker compose -f deploy/docker-compose.mcp.yml up -d +``` + +Caddy terminates TLS and proxies to the MCP server. Set `MCP_SERVER_URL=https://mcp.example.com` in the Compose file or environment so OIDC callbacks resolve correctly. + +Health checks: `GET /health` returns `{"status": "ok"}`. + +## MCP endpoint URL + +The public MCP URL depends on how you configure `MCP_SERVER_URL`: + +| `MCP_SERVER_URL` | Internal mount path | Public MCP endpoint | +| --- | --- | --- | +| `https://mcp.example.com` | `/mcp` | `https://mcp.example.com/mcp` | +| `https://mcp.example.com/airbyte` | `/` (path-stripped LB) | `https://mcp.example.com/airbyte` | + +When configuring a remote MCP client, use the **public MCP endpoint URL** from the table above. + +## Environment variables + +| Variable | Required | Description | +| --- | --- | --- | +| `AIRBYTE_MCP_ENV_FILE` | Recommended | Path to dotenv secrets file (Airbyte Cloud credentials, connector secrets) | +| `MCP_SERVER_URL` | Recommended | Public HTTPS base URL (used for OIDC callbacks and mount path) | +| `MCP_CORS_ORIGINS` | Optional | Comma-separated CORS origins (e.g. `https://app.example.com`) | +| `OIDC_CONFIG_URL` | Optional | Keycloak OIDC discovery URL (enables OAuth when all three OIDC vars are set) | +| `OIDC_CLIENT_ID` | Optional | OIDC client ID | +| `OIDC_CLIENT_SECRET` | Optional | OIDC client secret | +| `AIRBYTE_CLOUD_MCP_SAFE_MODE` | Optional | Default `1`; restrict destructive ops to session-created resources | +| `AIRBYTE_CLOUD_MCP_READONLY_MODE` | Optional | Default `0`; disable write tools when `1` | + +## Integrating with remote MCP clients + +Remote MCP clients connect over HTTPS using **Streamable HTTP** (with SSE fallback). Stdio-based servers cannot be used directly. + +### 1. Deploy and expose HTTPS + +1. Deploy `airbyte-mcp-http` (Docker image from `Dockerfile.mcp`, or run directly). +2. Put it behind a reverse proxy with a valid TLS certificate. +3. Confirm the endpoint responds: `curl -s https://your-host/health`. + +### 2. Register the server with your client + +In your remote MCP client's admin or integration settings: + +1. Add a new remote MCP server. +2. Enter your public MCP URL (for example, `https://mcp.example.com/mcp`). +3. Choose an authentication method (see below). + +### 3. Authentication options + +#### Option A: Bearer token (simplest) + +Use this when the MCP server is reachable only on your network or protected by your proxy, and you want the client to send Airbyte Cloud credentials on each request. + +1. Select **Bearer Token** authentication in your MCP client. +2. Provide an Airbyte Cloud API bearer token (or the token your deployment expects in the `Authorization` header). + +The server resolves Airbyte Cloud credentials from HTTP headers in this order: + +1. `Authorization` (bearer token) +2. `X-Airbyte-Cloud-Client-Id` / `X-Airbyte-Cloud-Client-Secret` +3. `X-Airbyte-Workspace-Id` +4. Environment variables from `AIRBYTE_MCP_ENV_FILE` + +If your MCP client supports custom headers, you can pass `X-Airbyte-Workspace-Id` and client credentials instead of a bearer token. + +#### Option B: OIDC (Keycloak) + +Use this to protect the MCP endpoint with OAuth. Configure Keycloak (or compatible OIDC provider) and set all three OIDC environment variables. + +Whitelist your MCP client's OAuth callback URLs in your OIDC provider. Consult your client's documentation for the exact redirect URIs to register. + +Set `MCP_SERVER_URL` to your public HTTPS URL before enabling OIDC so redirect URIs resolve correctly. + +#### Option C: Network-level protection + +If the MCP server has no OIDC and is exposed publicly, protect it with firewall rules, IP allowlists, or an API gateway in front of the endpoint. + +### 4. CORS (if needed) + +If your MCP client's OAuth or browser-based flows require CORS, set: + +```bash +export MCP_CORS_ORIGINS="https://app.example.com,https://admin.example.com" +``` + +Remote clients typically connect server-to-server, so CORS is often unnecessary for tool calls. Enable it if you see CORS errors during OAuth or connection setup. + +## Local development with Streamable HTTP + +Test the same transport used in production: + +```bash +poe mcp-serve-streamable-http +``` + +This starts the server on `http://127.0.0.1:8080/mcp`. Use the [MCP Inspector](https://github.com/modelcontextprotocol/inspector) to connect over HTTP for local testing. + +## Reverse proxy examples + +### Caddy (included in `deploy/`) + +See `deploy/Caddyfile` and `deploy/docker-compose.mcp.yml`. + +### nginx + +```nginx +server { + listen 443 ssl; + server_name mcp.example.com; + + location / { + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /health { + proxy_pass http://127.0.0.1:8080/health; + } +} +``` + +## Related documentation + +- [MCP server setup (stdio clients)](../airbyte/mcp/__init__.py) — local Claude Desktop / Cursor configuration +- [Contributing: MCP development](./CONTRIBUTING.md#mcp-server-development) diff --git a/pyproject.toml b/pyproject.toml index 96cd30ad7..fcd7021d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,6 +175,7 @@ fix-and-check = { shell = "poe fix && poe check" } mcp-serve-local = { cmd = "airbyte-mcp", help = "Start the MCP server with STDIO transport" } mcp-serve-http = { cmd = "python -c \"from airbyte.mcp.server import app; app.run(transport='http', host='127.0.0.1', port=8000)\"", help = "Start the MCP server with HTTP transport" } mcp-serve-sse = { cmd = "python -c \"from airbyte.mcp.server import app; app.run(transport='sse', host='127.0.0.1', port=8000)\"", help = "Start the MCP server with SSE transport" } +mcp-serve-streamable-http = { cmd = "airbyte-mcp-http", help = "Start the MCP server with Streamable HTTP transport on localhost:8080 (production transport)" } mcp-inspect = { cmd = "fastmcp inspect airbyte/mcp/server.py:app", help = "Inspect MCP tools and resources (supports --tools, --health, etc.)" } mcp-tool-test = { cmd = "python -m fastmcp_extensions.utils.test_tool --app airbyte.mcp.server:app", help = "Test MCP tools directly with JSON arguments: poe mcp-tool-test ''" } mcp-docs-md = { cmd = "python scripts/generate_mcp_markdown.py", help = "Generate Markdown docs for the MCP server into docs/mcp-generated/ (Docusaurus- and pdoc-compatible)" } diff --git a/tests/unit_tests/test_mcp_http_config.py b/tests/unit_tests/test_mcp_http_config.py new file mode 100644 index 000000000..c072b5613 --- /dev/null +++ b/tests/unit_tests/test_mcp_http_config.py @@ -0,0 +1,37 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""Unit tests for hosted MCP HTTP configuration.""" + +from __future__ import annotations + +from starlette.middleware.cors import CORSMiddleware + +from airbyte.mcp._http_config import build_http_middleware, parse_cors_origins + + +def test_parse_cors_origins_empty() -> None: + """Empty or whitespace-only input returns no origins.""" + assert parse_cors_origins("") == [] + assert parse_cors_origins(None) == [] + assert parse_cors_origins(" , , ") == [] + + +def test_parse_cors_origins_trims_and_splits() -> None: + """Origins are split on commas and trimmed.""" + assert parse_cors_origins("https://app.example.com, https://admin.example.com") == [ + "https://app.example.com", + "https://admin.example.com", + ] + + +def test_build_http_middleware_without_origins() -> None: + """No CORS middleware is added when origins are unset.""" + assert build_http_middleware(cors_origins="") == [] + + +def test_build_http_middleware_with_origins() -> None: + """CORS middleware is configured when origins are provided.""" + middleware = build_http_middleware(cors_origins="https://app.example.com") + assert len(middleware) == 1 + assert middleware[0].cls is CORSMiddleware + assert middleware[0].kwargs["allow_origins"] == ["https://app.example.com"] + assert "mcp-protocol-version" in middleware[0].kwargs["allow_headers"]