diff --git a/README.md b/README.md index 7bae6bb..4f98af6 100644 --- a/README.md +++ b/README.md @@ -337,6 +337,7 @@ More examples in the [`examples/`](examples/) directory: | Injects API key via network interception hook (TypeScript) | [`examples/typescript/network_interception/`](examples/typescript/network_interception/) | | Claude Code CLI in micro-VM with GitHub bootstrap | [`examples/claude-code/`](examples/claude-code/) | | Claude Code with Docker inside sandbox via SDK | [`examples/claude-code-with-docker/`](examples/claude-code-with-docker/) | +| Claude Code with Claude Pro/Max subscription in the sandbox | [`examples/claude-danger/`](examples/claude-danger/) | | OpenAI Codex CLI in micro-VM with GitHub bootstrap | [`examples/codex/`](examples/codex/) | | Docker daemon inside sandbox with systemd | [`examples/docker-in-sandbox/`](examples/docker-in-sandbox/) | | Streamlit chatbot using Agent Client Protocol | [`examples/agent-client-protocol/`](examples/agent-client-protocol/) | diff --git a/RELEASE.md b/RELEASE.md index efa2d34..5aecb61 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,5 +1,10 @@ # Release Notes +## 0.2.9 + +* Fixed Linux `--allow-host` DNS reachability for intercepted sandboxes by adding a host-side DNS forwarder and nftables redirection for guest DNS queries ([#94](https://github.com/jingkaihe/matchlock/issues/94), initial contribution by [@nemtsov](https://github.com/nemtsov)). +* Improved Linux interception reliability by binding proxy services to the sandbox gateway IP, applying secret-related allowed hosts before firewall setup, and falling back across configured DNS resolvers when an upstream fails. + ## 0.2.8 * Added custom secret placeholder support across `matchlock run` and the Go, Python, and TypeScript SDKs, including `--secret-placeholder`, `--secret-file`, and builder helpers for caller-defined in-VM placeholder values. diff --git a/VERSION.txt b/VERSION.txt index a45be46..1866a36 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -0.2.8 +0.2.9 diff --git a/examples/claude-danger/Dockerfile b/examples/claude-danger/Dockerfile new file mode 100644 index 0000000..3107684 --- /dev/null +++ b/examples/claude-danger/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV HOME=/home/agent +ENV PATH=/home/agent/.local/bin:/home/agent/.claude/local:$PATH + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + && useradd -m -s /bin/bash agent \ + && install -d -m 700 -o agent -g agent /home/agent/.claude \ + && install -d -m 755 -o agent -g agent /home/agent/workspace \ + && su agent -c 'HOME=/home/agent bash -lc "curl -fsSL https://claude.ai/install.sh | bash"' \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /home/agent diff --git a/examples/claude-danger/README.md b/examples/claude-danger/README.md new file mode 100644 index 0000000..a9aa4ac --- /dev/null +++ b/examples/claude-danger/README.md @@ -0,0 +1,50 @@ +# Claude Code with Claude Pro/Max Subscription in the Sandbox + +Run Claude Code inside a Matchlock sandbox using your local Claude subscription credentials, while keeping the real subscription token on the host and injecting it with HTTP interception. + +## Prerequisites + +Build the example image + +Using Docker: + +```bash +docker build -t claude-danger:latest examples/claude-danger +docker save claude-danger:latest | matchlock image import claude-danger:latest +``` + +Or using Matchlock: + +```bash +matchlock build -t claude-danger:latest --build-cache-size 30000 examples/claude-danger +``` + +Ensure you are logged into Claude locally and have `~/.claude/.credentials.json`. + +## Run + +From the repository root: + +```bash +./examples/claude-danger/claude-danger +``` + +Optional flags: + +```bash +./examples/claude-danger/claude-danger \ + --cpus 4 \ + --memory 8192 \ + --image claude-danger:latest \ + --credentials-file ~/.claude/.credentials.json \ + --allowed-host api.anthropic.com \ + --allowed-host platform.claude.com +``` + +The script uses `matchlock` from `PATH` by default. Override with `MATCHLOCK_BIN=/path/to/matchlock` if needed. + +## Notes + +- The image installs Claude Code with Anthropic's official installer. +- By default the helper launches Claude Code with `claude-opus-4-6` and `--effort high`. +- By default it allows all outbound hosts (`*`). Pass one or more `--allowed-host` flags to restrict egress. diff --git a/examples/claude-danger/claude-danger b/examples/claude-danger/claude-danger new file mode 100755 index 0000000..52c7ae8 --- /dev/null +++ b/examples/claude-danger/claude-danger @@ -0,0 +1,517 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.12" +# dependencies = ["click", "httpx", "matchlock"] +# /// +"""Launch Claude Code with host-side Claude subscription.""" + +from __future__ import annotations + +import json +import logging +import os +import shlex +import sys +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import IO, Any, cast + +import click +import httpx +from matchlock import ( + Client, + Config, + ImageConfig, + NetworkHookRequest, + NetworkHookRequestMutation, + NetworkHookResult, + NetworkHookRule, + NetworkInterceptionConfig, + Sandbox, +) + +try: + import termios as _termios_runtime + import tty as _tty_runtime +except ImportError: # pragma: no cover + raise RuntimeError("POSIX termios/tty support required") from None + +termios = _termios_runtime +tty = _tty_runtime + + +logging.basicConfig(format="%(levelname)s %(message)s", level=logging.INFO) +log = logging.getLogger(__name__) + +ANTHROPIC_CLIENT_ID = "9d1c250a-e61b-44d9-88ed-5944d1962f5e" +ANTHROPIC_TOKEN_ENDPOINT = "https://console.anthropic.com/v1/oauth/token" +TOKEN_REFRESH_THRESHOLD_MS = 10 * 60 * 1000 +GUEST_API_KEY_PLACEHOLDER = "sk-ant-api03-guest-placeholder" +GUEST_HOME = "/home/agent" +GUEST_WORKDIR = "/home/agent" +GUEST_CLAUDE_CONFIG_DIR = f"{GUEST_HOME}/.claude" +GUEST_CLAUDE_CONFIG_FILE = f"{GUEST_CLAUDE_CONFIG_DIR}/.config.json" +DEFAULT_CLAUDE_MODEL = "claude-opus-4-6" +DEFAULT_CLAUDE_EFFORT = "high" +DEFAULT_GUEST_CLAUDE_STATE: dict[str, object] = { + "numStartups": 1, + "theme": "dark", + "hasCompletedOnboarding": True, + "lastOnboardingVersion": "2.1.96", +} +DEFAULT_GUEST_CLAUDE_SETTINGS = { + "apiKeyHelper": f"printf %s {GUEST_API_KEY_PLACEHOLDER}", + "skipDangerousModePermissionPrompt": True, +} +DEFAULT_GUEST_PROJECT_STATE: dict[str, object] = { + "allowedTools": [], + "mcpContextUris": [], + "mcpServers": {}, + "enabledMcpjsonServers": [], + "disabledMcpjsonServers": [], + "exampleFiles": [], + "hasTrustDialogAccepted": True, + "hasCompletedProjectOnboarding": True, + "projectOnboardingSeenCount": 1, + "hasClaudeMdExternalIncludesApproved": False, + "hasClaudeMdExternalIncludesWarningShown": False, +} + + +class _FDReader(IO[bytes]): + def __init__(self, fd: int): + self._fd = fd + + def read(self, size: int = 4096) -> bytes: + chunk_size = size if size > 0 else 4096 + try: + return os.read(self._fd, chunk_size) + except OSError: + return b"" + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return False + + def seekable(self) -> bool: + return False + + def close(self) -> None: + return None + + @property + def closed(self) -> bool: + return False + + +@dataclass +class ClaudeOAuthCredentials: + access_token: str + refresh_token: str + expires_at_ms: int + scopes: list[str] + subscription_type: str | None = None + rate_limit_tier: str | None = None + + +class ClaudeOAuthProvider: + def __init__(self, credentials_path: Path): + self._credentials_path = credentials_path.expanduser() + self._lock = threading.Lock() + self._credentials = self._read_credentials() + + @property + def credentials_path(self) -> Path: + return self._credentials_path + + @property + def subscription_type(self) -> str: + return self._credentials.subscription_type or "max" + + @property + def rate_limit_tier(self) -> str | None: + return self._credentials.rate_limit_tier + + @property + def scopes(self) -> list[str]: + return list(self._credentials.scopes) + + def access_token(self) -> str: + with self._lock: + now_ms = int(time.time() * 1000) + if self._credentials.expires_at_ms <= now_ms + TOKEN_REFRESH_THRESHOLD_MS: + self._credentials = self._refresh_credentials(self._credentials) + self._write_credentials(self._credentials) + return self._credentials.access_token + + def _read_credentials(self) -> ClaudeOAuthCredentials: + try: + payload = json.loads(self._credentials_path.read_text()) + except FileNotFoundError as exc: + raise RuntimeError( + f"Claude credentials not found: {self._credentials_path}" + ) from exc + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Claude credentials file is not valid JSON: {self._credentials_path}" + ) from exc + + oauth = payload.get("claudeAiOauth") + if not isinstance(oauth, dict): + raise RuntimeError( + f"Claude credentials file is missing claudeAiOauth: {self._credentials_path}" + ) + + access_token = oauth.get("accessToken") + refresh_token = oauth.get("refreshToken") + expires_at_ms = oauth.get("expiresAt") + scopes = oauth.get("scopes") or [] + if not isinstance(access_token, str) or not access_token: + raise RuntimeError("Claude credentials file is missing accessToken") + if not isinstance(refresh_token, str) or not refresh_token: + raise RuntimeError("Claude credentials file is missing refreshToken") + if not isinstance(expires_at_ms, int): + raise RuntimeError("Claude credentials file is missing integer expiresAt") + if not isinstance(scopes, list) or not all( + isinstance(scope, str) for scope in scopes + ): + raise RuntimeError("Claude credentials file has invalid scopes") + + return ClaudeOAuthCredentials( + access_token=access_token, + refresh_token=refresh_token, + expires_at_ms=expires_at_ms, + scopes=scopes, + subscription_type=_optional_str(oauth.get("subscriptionType")), + rate_limit_tier=_optional_str(oauth.get("rateLimitTier")), + ) + + def _refresh_credentials( + self, creds: ClaudeOAuthCredentials + ) -> ClaudeOAuthCredentials: + log.info("refreshing Claude OAuth token from %s", ANTHROPIC_TOKEN_ENDPOINT) + try: + with httpx.Client(timeout=30) as client: + resp = client.post( + ANTHROPIC_TOKEN_ENDPOINT, + json={ + "grant_type": "refresh_token", + "client_id": ANTHROPIC_CLIENT_ID, + "refresh_token": creds.refresh_token, + }, + ) + resp.raise_for_status() + token_payload = resp.json() + except httpx.HTTPStatusError as exc: + raise RuntimeError( + f"Claude OAuth refresh failed with HTTP {exc.response.status_code}: {exc.response.text}" + ) from exc + except (httpx.HTTPError, ValueError) as exc: + raise RuntimeError(f"Claude OAuth refresh failed: {exc}") from exc + + access_token = token_payload.get("access_token") + expires_in = token_payload.get("expires_in") + if not isinstance(access_token, str) or not access_token: + raise RuntimeError("Claude OAuth refresh response is missing access_token") + if not isinstance(expires_in, int): + raise RuntimeError("Claude OAuth refresh response is missing expires_in") + + refresh_token = token_payload.get("refresh_token") + if not isinstance(refresh_token, str) or not refresh_token: + refresh_token = creds.refresh_token + + return ClaudeOAuthCredentials( + access_token=access_token, + refresh_token=refresh_token, + expires_at_ms=int(time.time() * 1000) + (expires_in * 1000), + scopes=creds.scopes, + subscription_type=creds.subscription_type, + rate_limit_tier=creds.rate_limit_tier, + ) + + def _write_credentials(self, creds: ClaudeOAuthCredentials) -> None: + payload: dict[str, Any] = { + "claudeAiOauth": { + "accessToken": creds.access_token, + "refreshToken": creds.refresh_token, + "expiresAt": creds.expires_at_ms, + "scopes": creds.scopes, + } + } + if creds.subscription_type: + payload["claudeAiOauth"]["subscriptionType"] = creds.subscription_type + if creds.rate_limit_tier: + payload["claudeAiOauth"]["rateLimitTier"] = creds.rate_limit_tier + + self._credentials_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = self._credentials_path.with_suffix( + self._credentials_path.suffix + ".tmp" + ) + temp_path.write_text(json.dumps(payload, indent=2) + "\n") + os.chmod(temp_path, 0o600) + temp_path.replace(self._credentials_path) + log.info("updated Claude OAuth credentials at %s", self._credentials_path) + + +def _optional_str(value: Any) -> str | None: + return value if isinstance(value, str) and value else None + + +def _pop_header(headers: dict[str, list[str]], name: str) -> list[str]: + values: list[str] = [] + for key in list(headers): + if key.lower() == name.lower(): + values.extend(headers.pop(key)) + return values + + +def _split_header_values(values: list[str]) -> list[str]: + parts: list[str] = [] + for value in values: + for item in value.split(","): + stripped = item.strip() + if stripped: + parts.append(stripped) + return parts + + +def guest_claude_state_json(working_dir: str) -> str: + payload = dict(DEFAULT_GUEST_CLAUDE_STATE) + payload["customApiKeyResponses"] = { + "approved": [GUEST_API_KEY_PLACEHOLDER[-20:]], + } + payload["projects"] = { + working_dir: dict(DEFAULT_GUEST_PROJECT_STATE), + } + return json.dumps(payload, indent=2) + "\n" + + +def guest_claude_settings_json() -> str: + return json.dumps(DEFAULT_GUEST_CLAUDE_SETTINGS, indent=2) + "\n" + + +def build_sandbox( + image: str, + cpus: float, + memory_mb: int, + allowed_hosts: tuple[str, ...], + provider: ClaudeOAuthProvider, +) -> Sandbox: + def before_hook(req: NetworkHookRequest) -> NetworkHookResult: + headers = { + key: list(values) for key, values in (req.request_headers or {}).items() + } + _pop_header(headers, "X-Api-Key") + headers["Authorization"] = [f"Bearer {provider.access_token()}"] + + beta_values = _split_header_values(_pop_header(headers, "anthropic-beta")) + if "oauth-2025-04-20" not in beta_values: + beta_values.append("oauth-2025-04-20") + headers["anthropic-beta"] = beta_values + + return NetworkHookResult( + action="mutate", + request=NetworkHookRequestMutation(headers=headers), + ) + + host_patterns = allowed_hosts if allowed_hosts else ("*",) + + return ( + Sandbox(image) + .with_cpus(cpus) + .with_memory(memory_mb) + .with_env("HOME", GUEST_HOME) + .with_env("CLAUDE_CONFIG_DIR", GUEST_CLAUDE_CONFIG_DIR) + .allow_host(*host_patterns) + .with_image_config( + ImageConfig( + user="agent", + working_dir=GUEST_WORKDIR, + ) + ) + .with_network_interception( + NetworkInterceptionConfig( + rules=[ + NetworkHookRule( + name="inject-claude-subscription-headers", + phase="before", + hosts=["api.anthropic.com", "platform.claude.com"], + hook=before_hook, + ) + ] + ) + ) + ) + + +def prepare_guest_claude_home( + client: Client, + guest_state_json: str, + guest_settings_json: str, +) -> None: + client.exec( + f"mkdir -p {GUEST_CLAUDE_CONFIG_DIR} && chown agent:agent {GUEST_CLAUDE_CONFIG_DIR} && chmod 700 {GUEST_CLAUDE_CONFIG_DIR}" + ) + client.write_file( + f"{GUEST_HOME}/.claude.json", + guest_state_json, + mode=0o644, + ) + client.exec( + f"chmod 644 {GUEST_HOME}/.claude.json" + ) + client.write_file( + GUEST_CLAUDE_CONFIG_FILE, + guest_state_json, + mode=0o644, + ) + client.exec( + f"chmod 644 {GUEST_CLAUDE_CONFIG_FILE}" + ) + client.write_file( + f"{GUEST_CLAUDE_CONFIG_DIR}/settings.json", + guest_settings_json, + mode=0o644, + ) + client.exec( + f"chmod 644 {GUEST_CLAUDE_CONFIG_DIR}/settings.json" + ) + client.exec(f"rm -f {GUEST_CLAUDE_CONFIG_DIR}/.credentials.json") + + +def run_interactive_shell(client: Client, model: str, effort: str) -> None: + if ( + not sys.stdin.isatty() + or not sys.stdout.isatty() + or termios is None + or tty is None + ): + raise RuntimeError("interactive terminal required (POSIX TTY)") + + fd = sys.stdin.fileno() + size = os.get_terminal_size(fd) + rows = size.lines + cols = size.columns + + old_state = termios.tcgetattr(fd) + tty.setraw(fd) + try: + stdin_reader = _FDReader(fd) + claude_command = shlex.join( + [ + "claude", + "--dangerously-skip-permissions", + "--model", + model, + "--effort", + effort, + ] + ) + result = client.exec_interactive( + ( + f"sh -lc '" + f"if [ \"$(id -un)\" = agent ]; then " + f"cd {GUEST_WORKDIR} && exec env HOME={GUEST_HOME} CLAUDE_CONFIG_DIR={GUEST_CLAUDE_CONFIG_DIR} {claude_command}; " + f"else " + f"exec su agent -c \"cd {GUEST_WORKDIR} && env HOME={GUEST_HOME} CLAUDE_CONFIG_DIR={GUEST_CLAUDE_CONFIG_DIR} {claude_command}\"; " + f"fi'" + ), + stdin=cast(IO[bytes], stdin_reader), + stdout=sys.stdout, + working_dir=GUEST_WORKDIR, + rows=rows, + cols=cols, + ) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_state) + + print(f"\nShell exited: code={result.exit_code} duration_ms={result.duration_ms}") + + +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--image", + default="claude-danger:latest", + show_default=True, + help="Sandbox image to run.", +) +@click.option( + "--credentials-file", + type=click.Path(path_type=Path, dir_okay=False, resolve_path=True), + default=Path.home() / ".claude" / ".credentials.json", + show_default=True, + help="Path to Claude credentials JSON on the host.", +) +@click.option( + "--cpus", + type=float, + default=2.0, + show_default=True, + help="vCPU count.", +) +@click.option( + "--memory", + "memory_mb", + type=int, + default=4096, + show_default=True, + help="Memory in MB.", +) +@click.option( + "--allowed-host", + "allowed_hosts", + multiple=True, + help="Host glob to allow. Repeatable. Defaults to '*' when unspecified.", +) +@click.option( + "--model", + default=DEFAULT_CLAUDE_MODEL, + show_default=True, + help="Claude model to launch.", +) +@click.option( + "--effort", + type=click.Choice(["low", "medium", "high", "max"], case_sensitive=False), + default=DEFAULT_CLAUDE_EFFORT, + show_default=True, + help="Claude reasoning effort.", +) +def main( + image: str, + credentials_file: Path, + cpus: float, + memory_mb: int, + allowed_hosts: tuple[str, ...], + model: str, + effort: str, +) -> None: + """Launch Claude Code in a sandbox using host-side Claude subscription.""" + + provider = ClaudeOAuthProvider(credentials_file) + guest_state_json = guest_claude_state_json(GUEST_WORKDIR) + guest_settings_json = guest_claude_settings_json() + + log.info("using Claude subscription credentials from %s", provider.credentials_path) + config = Config(binary_path=os.environ.get("MATCHLOCK_BIN", "matchlock")) + sandbox = build_sandbox(image, cpus, memory_mb, allowed_hosts, provider) + + client = Client(config) + try: + with client: + vm_id = client.create(sandbox.options()) + log.info("sandbox ready vm=%s image=%s", vm_id, image) + prepare_guest_claude_home(client, guest_state_json, guest_settings_json) + run_interactive_shell(client, model, effort.lower()) + finally: + try: + client.remove() + except Exception as exc: # noqa: BLE001 + log.warning("remove failed (ignored): %s", exc) + + +if __name__ == "__main__": + main() diff --git a/pkg/net/dns_forwarder.go b/pkg/net/dns_forwarder.go new file mode 100644 index 0000000..d5f37aa --- /dev/null +++ b/pkg/net/dns_forwarder.go @@ -0,0 +1,212 @@ +//go:build linux + +package net + +import ( + "fmt" + "net" + "sync" + "time" + + "github.com/jingkaihe/matchlock/internal/errx" +) + +// DNSForwarder relays guest DNS queries to upstream resolvers. +type DNSForwarder struct { + conn *net.UDPConn + dnsServers []string + + requests chan dnsRequest + stopCh chan struct{} + closeOnce sync.Once + wg sync.WaitGroup + + upstreamMu sync.Mutex + upstreams map[net.Conn]struct{} +} + +type dnsRequest struct { + query []byte + clientAddr *net.UDPAddr +} + +const ( + dnsUpstreamTimeout = 5 * time.Second + dnsPacketBufSize = 4096 + // Bound concurrent upstream exchanges so a guest cannot create an + // unbounded number of goroutines or sockets by flooding DNS traffic. + dnsWorkerCount = 32 + dnsQueueSize = 128 +) + +// NewDNSForwarder starts a UDP forwarder on bindAddr. +func NewDNSForwarder(bindAddr string, dnsServers []string) (*DNSForwarder, error) { + if len(dnsServers) == 0 { + return nil, errx.With(ErrListen, " no DNS servers configured for forwarder") + } + addr, err := net.ResolveUDPAddr("udp4", fmt.Sprintf("%s:0", bindAddr)) + if err != nil { + return nil, errx.With(ErrListen, " resolving DNS bind addr %s: %w", bindAddr, err) + } + conn, err := net.ListenUDP("udp4", addr) + if err != nil { + return nil, errx.With(ErrListen, " on DNS port %s: %w", addr, err) + } + d := &DNSForwarder{ + conn: conn, + dnsServers: append([]string(nil), dnsServers...), + requests: make(chan dnsRequest, dnsQueueSize), + stopCh: make(chan struct{}), + upstreams: make(map[net.Conn]struct{}), + } + d.wg.Add(1) + go d.serve() + for range dnsWorkerCount { + d.wg.Add(1) + go d.worker() + } + return d, nil +} + +// Port returns the ephemeral port chosen by the kernel. +func (d *DNSForwarder) Port() int { + return d.conn.LocalAddr().(*net.UDPAddr).Port +} + +// Close stops the server and releases the socket. +func (d *DNSForwarder) Close() error { + d.closeOnce.Do(func() { + close(d.stopCh) + _ = d.conn.Close() + d.closeUpstreams() + }) + d.wg.Wait() + return nil +} + +func (d *DNSForwarder) serve() { + defer d.wg.Done() + buf := make([]byte, dnsPacketBufSize) + for { + n, clientAddr, err := d.conn.ReadFromUDP(buf) + if err != nil { + if d.isStopping() { + return + } + continue + } + + query := make([]byte, n) + copy(query, buf[:n]) + select { + case d.requests <- dnsRequest{query: query, clientAddr: clientAddr}: + case <-d.stopCh: + return + } + } +} + +func (d *DNSForwarder) worker() { + defer d.wg.Done() + for { + select { + case req := <-d.requests: + d.forward(req.query, req.clientAddr) + case <-d.stopCh: + return + } + } +} + +func (d *DNSForwarder) forward(query []byte, clientAddr *net.UDPAddr) { + for _, server := range d.dnsServers { + if d.isStopping() { + return + } + resp, err := d.exchange(query, server) + if err != nil { + continue + } + if d.isStopping() { + return + } + + _, _ = d.conn.WriteToUDP(resp, clientAddr) + return + } +} + +func (d *DNSForwarder) exchange(query []byte, server string) ([]byte, error) { + if d.isStopping() { + return nil, net.ErrClosed + } + upstream, err := net.DialTimeout("udp", dnsServerAddr(server), dnsUpstreamTimeout) + if err != nil { + return nil, err + } + if !d.trackUpstream(upstream) { + _ = upstream.Close() + return nil, net.ErrClosed + } + defer d.untrackUpstream(upstream) + defer upstream.Close() + + _ = upstream.SetDeadline(time.Now().Add(dnsUpstreamTimeout)) + if _, err := upstream.Write(query); err != nil { + return nil, err + } + + resp := make([]byte, dnsPacketBufSize) + n, err := upstream.Read(resp) + if err != nil { + return nil, err + } + + return resp[:n], nil +} + +func (d *DNSForwarder) isStopping() bool { + select { + case <-d.stopCh: + return true + default: + return false + } +} + +func (d *DNSForwarder) trackUpstream(upstream net.Conn) bool { + d.upstreamMu.Lock() + defer d.upstreamMu.Unlock() + + if d.isStopping() { + return false + } + d.upstreams[upstream] = struct{}{} + return true +} + +func (d *DNSForwarder) untrackUpstream(upstream net.Conn) { + d.upstreamMu.Lock() + defer d.upstreamMu.Unlock() + delete(d.upstreams, upstream) +} + +func (d *DNSForwarder) closeUpstreams() { + d.upstreamMu.Lock() + upstreams := make([]net.Conn, 0, len(d.upstreams)) + for upstream := range d.upstreams { + upstreams = append(upstreams, upstream) + } + d.upstreamMu.Unlock() + + for _, upstream := range upstreams { + _ = upstream.Close() + } +} + +func dnsServerAddr(server string) string { + if _, _, err := net.SplitHostPort(server); err == nil { + return server + } + return net.JoinHostPort(server, "53") +} diff --git a/pkg/net/dns_forwarder_test.go b/pkg/net/dns_forwarder_test.go new file mode 100644 index 0000000..2701ec8 --- /dev/null +++ b/pkg/net/dns_forwarder_test.go @@ -0,0 +1,256 @@ +//go:build linux + +package net + +import ( + "encoding/binary" + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// buildQuery crafts a minimal A/AAAA query for hostname. +func buildQuery(id uint16, qtype uint16, hostname string) []byte { + q := make([]byte, 0, 32) + var header [12]byte + binary.BigEndian.PutUint16(header[0:2], id) + header[2] = 0x01 // RD=1 + header[3] = 0x00 + binary.BigEndian.PutUint16(header[4:6], 1) // QDCOUNT + q = append(q, header[:]...) + + for _, label := range splitLabels(hostname) { + q = append(q, byte(len(label))) + q = append(q, []byte(label)...) + } + q = append(q, 0x00) // null terminator + + var tail [4]byte + binary.BigEndian.PutUint16(tail[0:2], qtype) + binary.BigEndian.PutUint16(tail[2:4], 1) // class IN + q = append(q, tail[:]...) + return q +} + +func splitLabels(s string) []string { + var out []string + start := 0 + for i := 0; i <= len(s); i++ { + if i == len(s) || s[i] == '.' { + if i > start { + out = append(out, s[start:i]) + } + start = i + 1 + } + } + return out +} + +func buildResponse(query []byte, answerIP net.IP) []byte { + resp := append([]byte(nil), query...) + resp[2] = 0x81 // QR=1, RD=1 + resp[3] = 0x80 // RA=1 + binary.BigEndian.PutUint16(resp[6:8], 1) + + answer := make([]byte, 16) + answer[0] = 0xc0 + answer[1] = 0x0c + binary.BigEndian.PutUint16(answer[2:4], 1) + binary.BigEndian.PutUint16(answer[4:6], 1) + binary.BigEndian.PutUint32(answer[6:10], 60) + binary.BigEndian.PutUint16(answer[10:12], 4) + copy(answer[12:16], answerIP.To4()) + + return append(resp, answer...) +} + +func startStubDNSServer(t *testing.T, ip net.IP) string { + t.Helper() + + ln, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + require.NoError(t, err) + + t.Cleanup(func() { + _ = ln.Close() + }) + + go func() { + buf := make([]byte, dnsPacketBufSize) + for { + n, addr, err := ln.ReadFromUDP(buf) + if err != nil { + return + } + resp := buildResponse(buf[:n], ip) + _, _ = ln.WriteToUDP(resp, addr) + } + }() + + return ln.LocalAddr().String() +} + +func startBlackholeDNSServer(t *testing.T, seen chan<- struct{}) string { + t.Helper() + + ln, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}) + require.NoError(t, err) + + t.Cleanup(func() { + _ = ln.Close() + }) + + go func() { + buf := make([]byte, dnsPacketBufSize) + for { + _, _, err := ln.ReadFromUDP(buf) + if err != nil { + return + } + select { + case seen <- struct{}{}: + default: + } + } + }() + + return ln.LocalAddr().String() +} + +func responseAnswerIPv4(t *testing.T, resp []byte) net.IP { + t.Helper() + + require.GreaterOrEqual(t, len(resp), 12, "response too short") + ancount := binary.BigEndian.Uint16(resp[6:8]) + require.NotZero(t, ancount, "expected at least 1 answer") + + off := 12 + for off < len(resp) && resp[off] != 0 { + off += 1 + int(resp[off]) + } + off += 5 // null terminator + qtype(2) + qclass(2) + require.LessOrEqual(t, off+12, len(resp), "response too short to contain answer record") + off += 2 + 2 + 2 + 4 + rdlen := binary.BigEndian.Uint16(resp[off : off+2]) + off += 2 + require.Equal(t, uint16(4), rdlen, "expected IPv4 answer") + require.LessOrEqual(t, off+4, len(resp), "response too short for IPv4 answer") + + return net.IPv4(resp[off], resp[off+1], resp[off+2], resp[off+3]) +} + +func TestDNSForwarderEndToEnd(t *testing.T) { + serverIP := net.IPv4(203, 0, 113, 9) + server := startStubDNSServer(t, serverIP) + d, err := NewDNSForwarder("127.0.0.1", []string{server}) + require.NoError(t, err) + defer d.Close() + + conn, err := net.DialUDP("udp4", nil, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: d.Port()}) + require.NoError(t, err) + defer conn.Close() + + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + _, err = conn.Write(buildQuery(0x4242, 1, "google.com")) + require.NoError(t, err) + buf := make([]byte, 512) + n, err := conn.Read(buf) + require.NoError(t, err) + require.Equal(t, uint16(0x4242), binary.BigEndian.Uint16(buf[0:2]), "id not echoed") + ip := responseAnswerIPv4(t, buf[:n]) + require.True(t, ip.Equal(serverIP), fmt.Sprintf("expected %s, got %s", serverIP, ip)) +} + +func TestDNSForwarderFallsBackAcrossServers(t *testing.T) { + serverIP := net.IPv4(198, 51, 100, 7) + server := startStubDNSServer(t, serverIP) + d, err := NewDNSForwarder("127.0.0.1", []string{"127.0.0.1:1", server}) + require.NoError(t, err) + defer d.Close() + + resp, err := d.exchange(buildQuery(0x5151, 1, "example.com"), "127.0.0.1:1") + require.Error(t, err) + require.Nil(t, resp) + + conn, err := net.DialUDP("udp4", nil, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: d.Port()}) + require.NoError(t, err) + defer conn.Close() + + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + _, err = conn.Write(buildQuery(0x6161, 1, "example.com")) + require.NoError(t, err) + + buf := make([]byte, dnsPacketBufSize) + n, err := conn.Read(buf) + require.NoError(t, err) + require.GreaterOrEqual(t, n, 12) + require.Equal(t, uint16(0x6161), binary.BigEndian.Uint16(buf[0:2])) +} + +func TestDNSForwarderPreservesResolverOrder(t *testing.T) { + firstIP := net.IPv4(203, 0, 113, 10) + secondIP := net.IPv4(203, 0, 113, 11) + first := startStubDNSServer(t, firstIP) + second := startStubDNSServer(t, secondIP) + d, err := NewDNSForwarder("127.0.0.1", []string{first, second}) + require.NoError(t, err) + defer d.Close() + + conn, err := net.DialUDP("udp4", nil, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: d.Port()}) + require.NoError(t, err) + defer conn.Close() + + for i := range 4 { + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + _, err = conn.Write(buildQuery(uint16(0x7000+i), 1, "example.internal")) + require.NoError(t, err) + + buf := make([]byte, dnsPacketBufSize) + n, err := conn.Read(buf) + require.NoError(t, err) + ip := responseAnswerIPv4(t, buf[:n]) + require.True(t, ip.Equal(firstIP), fmt.Sprintf("expected %s, got %s", firstIP, ip)) + } +} + +func TestDNSForwarderCloseInterruptsInFlightExchange(t *testing.T) { + seen := make(chan struct{}, 1) + server := startBlackholeDNSServer(t, seen) + d, err := NewDNSForwarder("127.0.0.1", []string{server}) + require.NoError(t, err) + t.Cleanup(func() { + _ = d.Close() + }) + + conn, err := net.DialUDP("udp4", nil, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: d.Port()}) + require.NoError(t, err) + defer conn.Close() + + _, err = conn.Write(buildQuery(0x7878, 1, "example.com")) + require.NoError(t, err) + + select { + case <-seen: + case <-time.After(2 * time.Second): + t.Fatal("forwarder did not send query to upstream") + } + + closed := make(chan struct{}) + go func() { + _ = d.Close() + close(closed) + }() + + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("Close blocked waiting for upstream timeout") + } +} + +func TestDNSForwarderRejectsNoServers(t *testing.T) { + _, err := NewDNSForwarder("127.0.0.1", nil) + require.Error(t, err) +} diff --git a/pkg/net/nftables.go b/pkg/net/nftables.go index 024338b..6887f82 100644 --- a/pkg/net/nftables.go +++ b/pkg/net/nftables.go @@ -16,23 +16,26 @@ const ( tableName = "matchlock" chainPreNAT = "prerouting" chainFwd = "forward" + chainInput = "input" + chainOutput = "output" ) type NFTablesRules struct { - tapInterface string - gatewayIP net.IP - httpPort uint16 - httpsPort uint16 - passthroughPort uint16 - dnsServers []net.IP - conn *nftables.Conn - table *nftables.Table + tapInterface string + gatewayIP net.IP + httpPort uint16 + httpsPort uint16 + passthroughPort uint16 + dnsForwarderPort uint16 + dnsServers []net.IP + conn *nftables.Conn + table *nftables.Table } func NewNFTablesRules(tapInterface, gatewayIP string, httpPort, httpsPort, passthroughPort int, dnsServers []string) *NFTablesRules { var dnsIPs []net.IP for _, s := range dnsServers { - if ip := net.ParseIP(s).To4(); ip != nil { + if ip := parseDNSIPv4(s); ip != nil { dnsIPs = append(dnsIPs, ip) } } @@ -46,6 +49,10 @@ func NewNFTablesRules(tapInterface, gatewayIP string, httpPort, httpsPort, passt } } +func (r *NFTablesRules) SetDNSForwarderPort(port int) { + r.dnsForwarderPort = uint16(port) +} + func (r *NFTablesRules) Setup() error { conn, err := nftables.New() if err != nil { @@ -74,6 +81,25 @@ func (r *NFTablesRules) Setup() error { Priority: nftables.ChainPriorityFilter, }) + var inputChain *nftables.Chain + var outputChain *nftables.Chain + if r.dnsForwarderPort > 0 { + inputChain = conn.AddChain(&nftables.Chain{ + Name: chainInput, + Table: r.table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookInput, + Priority: nftables.ChainPriorityFilter, + }) + outputChain = conn.AddChain(&nftables.Chain{ + Name: chainOutput, + Table: r.table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookOutput, + Priority: nftables.ChainPriorityFilter, + }) + } + conn.AddRule(&nftables.Rule{ Table: r.table, Chain: preChain, @@ -94,9 +120,28 @@ func (r *NFTablesRules) Setup() error { }) } - // Allow DNS (UDP port 53) from the VM to configured resolvers only. - // Must come before the UDP drop rule. Restricting destination IPs - // prevents DNS tunneling to attacker-controlled nameservers. + if r.dnsForwarderPort > 0 { + conn.AddRule(&nftables.Rule{ + Table: r.table, + Chain: preChain, + Exprs: r.buildDNSForwarderDNATRule(r.dnsForwarderPort), + }) + + conn.AddRule(&nftables.Rule{ + Table: r.table, + Chain: inputChain, + Exprs: r.buildDNSForwarderInputAcceptRule(r.dnsForwarderPort), + }) + + for _, dnsIP := range r.dnsServers { + conn.AddRule(&nftables.Rule{ + Table: r.table, + Chain: outputChain, + Exprs: r.buildDNSForwarderOutputAcceptRule(dnsIP), + }) + } + } + for _, dnsIP := range r.dnsServers { conn.AddRule(&nftables.Rule{ Table: r.table, @@ -225,8 +270,6 @@ func (r *NFTablesRules) buildForwardRule(isInput bool) []expr.Any { } } -// buildUDPDNSAcceptRule accepts UDP port 53 traffic from the TAP interface -// to a specific destination IP (e.g. 8.8.8.8). func (r *NFTablesRules) buildUDPDNSAcceptRule(dstIP net.IP) []expr.Any { return []expr.Any{ &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, @@ -241,7 +284,6 @@ func (r *NFTablesRules) buildUDPDNSAcceptRule(dstIP net.IP) []expr.Any { Register: 1, Data: []byte{unix.IPPROTO_UDP}, }, - // Match destination IP &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, @@ -253,7 +295,122 @@ func (r *NFTablesRules) buildUDPDNSAcceptRule(dstIP net.IP) []expr.Any { Register: 1, Data: dstIP.To4(), }, - // Match destination port 53 + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(53), + }, + &expr.Verdict{Kind: expr.VerdictAccept}, + } +} + +func (r *NFTablesRules) buildDNSForwarderDNATRule(dstPort uint16) []expr.Any { + return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.tapInterface), + }, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.IPPROTO_UDP}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(53), + }, + &expr.Immediate{ + Register: 1, + Data: r.gatewayIP, + }, + &expr.Immediate{ + Register: 2, + Data: binaryutil.BigEndian.PutUint16(dstPort), + }, + &expr.NAT{ + Type: expr.NATTypeDestNAT, + Family: unix.NFPROTO_IPV4, + RegAddrMin: 1, + RegProtoMin: 2, + }, + } +} + +func (r *NFTablesRules) buildDNSForwarderInputAcceptRule(dstPort uint16) []expr.Any { + return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: ifname(r.tapInterface), + }, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.IPPROTO_UDP}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 16, + Len: 4, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: r.gatewayIP, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 2, + Len: 2, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: binaryutil.BigEndian.PutUint16(dstPort), + }, + &expr.Verdict{Kind: expr.VerdictAccept}, + } +} + +func (r *NFTablesRules) buildDNSForwarderOutputAcceptRule(dstIP net.IP) []expr.Any { + return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.IPPROTO_UDP}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 16, + Len: 4, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: dstIP.To4(), + }, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseTransportHeader, @@ -320,6 +477,18 @@ func ifname(n string) []byte { return b } +func parseDNSIPv4(server string) net.IP { + if ip := net.ParseIP(server).To4(); ip != nil { + return ip + } + + host, _, err := net.SplitHostPort(server) + if err != nil { + return nil + } + return net.ParseIP(host).To4() +} + type NFTablesNAT struct { tapInterface string conn *nftables.Conn diff --git a/pkg/net/nftables_test.go b/pkg/net/nftables_test.go new file mode 100644 index 0000000..57e5bfe --- /dev/null +++ b/pkg/net/nftables_test.go @@ -0,0 +1,94 @@ +//go:build linux + +package net + +import ( + "net" + "testing" + + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestNewNFTablesRulesParsesDNSIPv4Servers(t *testing.T) { + rules := NewNFTablesRules( + "tap0", + "192.168.100.1", + 8080, + 8443, + 0, + []string{"1.1.1.1", "8.8.8.8:53", "resolver.local", "[2606:4700:4700::1111]:53"}, + ) + + require.Equal(t, []net.IP{ + net.IPv4(1, 1, 1, 1).To4(), + net.IPv4(8, 8, 8, 8).To4(), + }, rules.dnsServers) +} + +func TestBuildDNSForwarderInputAcceptRule(t *testing.T) { + rules := &NFTablesRules{ + tapInterface: "tap0", + gatewayIP: net.IPv4(192, 168, 100, 1).To4(), + } + + exprs := rules.buildDNSForwarderInputAcceptRule(5353) + require.Len(t, exprs, 9) + + iifMeta, ok := exprs[0].(*expr.Meta) + require.True(t, ok) + require.Equal(t, expr.MetaKeyIIFNAME, iifMeta.Key) + + iifCmp, ok := exprs[1].(*expr.Cmp) + require.True(t, ok) + require.Equal(t, ifname("tap0"), iifCmp.Data) + + protoMeta, ok := exprs[2].(*expr.Meta) + require.True(t, ok) + require.Equal(t, expr.MetaKeyL4PROTO, protoMeta.Key) + + protoCmp, ok := exprs[3].(*expr.Cmp) + require.True(t, ok) + require.Equal(t, []byte{unix.IPPROTO_UDP}, protoCmp.Data) + + dstIPCmp, ok := exprs[5].(*expr.Cmp) + require.True(t, ok) + require.Equal(t, []byte(net.IPv4(192, 168, 100, 1).To4()), dstIPCmp.Data) + + dstPortCmp, ok := exprs[7].(*expr.Cmp) + require.True(t, ok) + require.Equal(t, binaryutil.BigEndian.PutUint16(5353), dstPortCmp.Data) + + verdict, ok := exprs[8].(*expr.Verdict) + require.True(t, ok) + require.Equal(t, expr.VerdictAccept, verdict.Kind) +} + +func TestBuildDNSForwarderOutputAcceptRule(t *testing.T) { + rules := &NFTablesRules{} + + exprs := rules.buildDNSForwarderOutputAcceptRule(net.IPv4(8, 8, 8, 8)) + require.Len(t, exprs, 7) + + protoMeta, ok := exprs[0].(*expr.Meta) + require.True(t, ok) + require.Equal(t, expr.MetaKeyL4PROTO, protoMeta.Key) + + protoCmp, ok := exprs[1].(*expr.Cmp) + require.True(t, ok) + require.Equal(t, []byte{unix.IPPROTO_UDP}, protoCmp.Data) + + dstIPCmp, ok := exprs[3].(*expr.Cmp) + require.True(t, ok) + require.Equal(t, []byte(net.IPv4(8, 8, 8, 8).To4()), dstIPCmp.Data) + + dstPortCmp, ok := exprs[5].(*expr.Cmp) + require.True(t, ok) + require.Equal(t, binaryutil.BigEndian.PutUint16(53), dstPortCmp.Data) + + verdict, ok := exprs[6].(*expr.Verdict) + require.True(t, ok) + require.Equal(t, expr.VerdictAccept, verdict.Kind) +} diff --git a/pkg/sandbox/sandbox_linux.go b/pkg/sandbox/sandbox_linux.go index abbe57d..969f2bf 100644 --- a/pkg/sandbox/sandbox_linux.go +++ b/pkg/sandbox/sandbox_linux.go @@ -35,6 +35,7 @@ type Sandbox struct { config *api.Config machine vm.Machine proxy *sandboxnet.TransparentProxy + dnsForwarder *sandboxnet.DNSForwarder fwRules FirewallRules natRules *sandboxnet.NFTablesNAT policy *policy.Engine @@ -254,6 +255,21 @@ func New(ctx context.Context, config *api.Config, opts *Options) (sb *Sandbox, r subnetCIDR = subnetInfo.GatewayIP + "/24" } + if config.Network != nil && len(config.Network.Secrets) > 0 { + hostSet := make(map[string]bool) + for _, h := range config.Network.AllowedHosts { + hostSet[h] = true + } + for _, secret := range config.Network.Secrets { + for _, h := range secret.Hosts { + if !hostSet[h] { + config.Network.AllowedHosts = append(config.Network.AllowedHosts, h) + hostSet[h] = true + } + } + } + } + vmConfig := &vm.VMConfig{ ID: id, KernelPath: kernelPath, @@ -301,22 +317,6 @@ func New(ctx context.Context, config *api.Config, opts *Options) (sb *Sandbox, r }) } - // Auto-add secret hosts to allowed hosts if secrets are defined - if config.Network != nil && len(config.Network.Secrets) > 0 { - hostSet := make(map[string]bool) - for _, h := range config.Network.AllowedHosts { - hostSet[h] = true - } - for _, secret := range config.Network.Secrets { - for _, h := range secret.Hosts { - if !hostSet[h] { - config.Network.AllowedHosts = append(config.Network.AllowedHosts, h) - hostSet[h] = true - } - } - } - } - overlaySnapshots, err := prepareOverlaySnapshots(config, stateMgr.Dir(id)) if err != nil { machine.Close(ctx) @@ -332,15 +332,20 @@ func New(ctx context.Context, config *api.Config, opts *Options) (sb *Sandbox, r // Create event channel events := make(chan api.Event, 100) - // Set up transparent proxy for HTTP/HTTPS interception - const proxyBindAddr = "0.0.0.0" - var proxy *sandboxnet.TransparentProxy + var dnsForwarder *sandboxnet.DNSForwarder var fwRules FirewallRules if needsProxy { + if gatewayIP == "" { + machine.Close(ctx) + releaseSubnet() + stateMgr.Unregister(id) + return nil, errx.With(ErrCreateProxy, ": missing gateway IP for proxy bind") + } + proxy, err = sandboxnet.NewTransparentProxy(&sandboxnet.ProxyConfig{ - BindAddr: proxyBindAddr, + BindAddr: gatewayIP, HTTPPort: 0, HTTPSPort: 0, PassthroughPort: 0, @@ -357,8 +362,20 @@ func New(ctx context.Context, config *api.Config, opts *Options) (sb *Sandbox, r proxy.Start() - fwRules = sandboxnet.NewNFTablesRules(linuxMachine.TapName(), gatewayIP, proxy.HTTPPort(), proxy.HTTPSPort(), proxy.PassthroughPort(), config.Network.GetDNSServers()) + dnsForwarder, err = sandboxnet.NewDNSForwarder(gatewayIP, config.Network.GetDNSServers()) + if err != nil { + proxy.Close() + machine.Close(ctx) + releaseSubnet() + stateMgr.Unregister(id) + return nil, errx.Wrap(ErrCreateProxy, err) + } + + nfRules := sandboxnet.NewNFTablesRules(linuxMachine.TapName(), gatewayIP, proxy.HTTPPort(), proxy.HTTPSPort(), proxy.PassthroughPort(), config.Network.GetDNSServers()) + nfRules.SetDNSForwarderPort(dnsForwarder.Port()) + fwRules = nfRules if err := fwRules.Setup(); err != nil { + dnsForwarder.Close() proxy.Close() machine.Close(ctx) releaseSubnet() @@ -402,6 +419,9 @@ func New(ctx context.Context, config *api.Config, opts *Options) (sb *Sandbox, r if proxy != nil { proxy.Close() } + if dnsForwarder != nil { + dnsForwarder.Close() + } if fwRules != nil { fwRules.Cleanup() } @@ -417,6 +437,7 @@ func New(ctx context.Context, config *api.Config, opts *Options) (sb *Sandbox, r config: config, machine: machine, proxy: proxy, + dnsForwarder: dnsForwarder, fwRules: fwRules, natRules: natRules, policy: policyEngine, @@ -643,6 +664,11 @@ func (s *Sandbox) Close(ctx context.Context) error { markCleanup("proxy_close", nil) } + if s.dnsForwarder != nil { + _ = s.dnsForwarder.Close() + s.dnsForwarder = nil + } + // Release subnet allocation if s.subnetAlloc != nil { if err := s.subnetAlloc.Release(s.id); err != nil {