diff --git a/python/semantic_kernel/connectors/openapi_plugin/server_url_validator.py b/python/semantic_kernel/connectors/openapi_plugin/server_url_validator.py index a3ede15f0f3c..5c03195fb7fa 100644 --- a/python/semantic_kernel/connectors/openapi_plugin/server_url_validator.py +++ b/python/semantic_kernel/connectors/openapi_plugin/server_url_validator.py @@ -16,6 +16,40 @@ DEFAULT_ALLOWED_SCHEME = "https" +# Cloud metadata / credential endpoints. Most fall inside ranges `_try_classify_ipv4` already +# rejects, but `168.63.129.16` (Azure WireServer) is publicly routable, so no range check reaches +# it. They are listed explicitly and checked even when `allow_private_network_access` is set: +# reaching a host on your own network and reaching the instance's credential endpoint are +# different requests, and only the first is what that option is for. +CLOUD_METADATA_ADDRESSES: frozenset[ipaddress.IPv4Address | ipaddress.IPv6Address] = frozenset( + ipaddress.ip_address(address) + for address in ( + "169.254.169.254", # AWS IMDS, GCP, Azure, OCI, DigitalOcean, Hetzner, OpenStack + "169.254.170.2", # AWS ECS task IAM role credentials + "169.254.170.23", # AWS EKS Pod Identity Agent + "168.63.129.16", # Azure WireServer / platform channel (publicly routable) + "100.100.100.200", # Alibaba Cloud + "192.0.0.192", # Oracle Cloud (Classic) + "169.254.42.42", # Scaleway + "fd00:ec2::254", # AWS IMDS over IPv6 + "fd00:ec2::23", # AWS EKS Pod Identity Agent over IPv6 + ) +) + +# NAT64 (RFC 6052) and 6to4 (RFC 3056) carry an IPv4 address inside the IPv6 one. Only the /96 +# embedding is decoded: it is the only length the well-known prefix allows, and guessing the +# shorter lengths inside the RFC 8215 local-use prefix reads bytes that are not the embedded +# address, which would reject legitimate NAT64 targets. +_NAT64_IPV4_OFFSETS = (12, 13, 14, 15) +_NAT64_NETWORKS: tuple[ipaddress.IPv6Network, ...] = (ipaddress.IPv6Network("64:ff9b::/96"),) +# RFC 8215 local-use space, where the translator's prefix length is configuration rather than +# anything the address carries (RFC 6052 section 3.3). Bytes 12-15 hold the embedded address only +# for a /96; for the shorter lengths they hold the suffix, which the RFC says SHOULD be zero, so +# decoding them here reads 0.0.0.0 out of a perfectly ordinary target. Blocking the prefix whole +# avoids guessing in either direction: no public host is rejected as "unspecified", and no +# metadata address reaches the fit check through a length this code cannot determine. +_NAT64_LOCAL_USE_NETWORK = ipaddress.IPv6Network("64:ff9b:1::/48") + class ServerUrlValidationOptions(KernelBaseModel): """Options for validating OpenAPI operation request URLs.""" @@ -58,10 +92,17 @@ async def validate_server_url( "To allow this URL, add it to server_url_validation_allowed_base_urls." ) - if options.allow_private_network_access: - return + await _ensure_public_host( + parsed_url, dns_resolver, allow_private_network_access=options.allow_private_network_access + ) + - await _ensure_public_host(parsed_url, dns_resolver) +def is_cloud_metadata_address(address: str | ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """Return whether an address, or an IPv4 embedded in it, is a cloud metadata endpoint.""" + ip_address = ipaddress.ip_address(address) + if ip_address in CLOUD_METADATA_ADDRESSES: + return True + return any(embedded in CLOUD_METADATA_ADDRESSES for embedded in _embedded_ipv4s(ip_address)) def try_categorize_non_public_address( @@ -76,7 +117,45 @@ def try_categorize_non_public_address( if isinstance(ip_address, ipaddress.IPv4Address): return _try_classify_ipv4(ip_address) - return _try_classify_ipv6(ip_address) + blocked, category = _try_classify_ipv6(ip_address) + if blocked: + return blocked, category + + # 6to4, NAT64 and Teredo carry an IPv4 target inside an otherwise public-looking IPv6 + # address. Decode those and classify the IPv4 they name. + for embedded in _embedded_ipv4s(ip_address): + blocked, category = _try_classify_ipv4(embedded) + if blocked: + return blocked, f"{category} (embedded in IPv6)" + + return False, "" + + +def _embedded_ipv4s(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> list[ipaddress.IPv4Address]: + """Decode the IPv4 addresses carried inside IPv4-mapped, NAT64, 6to4 and Teredo IPv6 forms.""" + if not isinstance(address, ipaddress.IPv6Address): + return [] + + packed = address.packed + candidates: list[ipaddress.IPv4Address] = [] + + if address.ipv4_mapped is not None: + candidates.append(address.ipv4_mapped) + + if any(address in network for network in _NAT64_NETWORKS): + candidates.append(ipaddress.IPv4Address(bytes(packed[offset] for offset in _NAT64_IPV4_OFFSETS))) + + # `sixtofour` and `teredo` return None outside their own prefixes, so they carry the + # membership check with them and leave no offset arithmetic here to get wrong. Teredo in + # particular obfuscates the client IPv4 by XOR-ing the low 32 bits with all ones (RFC 4380), + # which is the kind of detail worth taking from the standard library rather than restating. + if address.sixtofour is not None: + candidates.append(address.sixtofour) + + if address.teredo is not None: + candidates.append(address.teredo[1]) + + return candidates def _parse_absolute_url(url: str, option_name: str = "url") -> ParseResult: @@ -127,7 +206,9 @@ def _matches_path_prefix(url_path: str, base_path: str) -> bool: return url_path.lower().startswith(base_path_with_slash.lower()) -async def _ensure_public_host(parsed_url: ParseResult, dns_resolver: DnsResolver | None) -> None: +async def _ensure_public_host( + parsed_url: ParseResult, dns_resolver: DnsResolver | None, allow_private_network_access: bool = False +) -> None: host = parsed_url.hostname if host is None: raise FunctionExecutionException(f"The request URI '{parsed_url.geturl()}' does not contain a valid host.") @@ -137,7 +218,7 @@ async def _ensure_public_host(parsed_url: ParseResult, dns_resolver: DnsResolver except ValueError: addresses = await _resolve_host(host, dns_resolver) else: - _ensure_public_address(parsed_url.geturl(), ip_address) + _ensure_public_address(parsed_url.geturl(), ip_address, allow_private_network_access) return if not addresses: @@ -147,7 +228,7 @@ async def _ensure_public_host(parsed_url: ParseResult, dns_resolver: DnsResolver ) for address in addresses: - _ensure_public_address(parsed_url.geturl(), address) + _ensure_public_address(parsed_url.geturl(), address, allow_private_network_access) async def _resolve_host( @@ -180,7 +261,18 @@ async def _resolve_host( return addresses -def _ensure_public_address(url: str, address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> None: +def _ensure_public_address( + url: str, address: ipaddress.IPv4Address | ipaddress.IPv6Address, allow_private_network_access: bool = False +) -> None: + if is_cloud_metadata_address(address): + raise FunctionExecutionException( + f"The request URI '{url}' is not allowed: host resolves to a cloud metadata endpoint ({address}), " + "which is blocked to prevent Server-Side Request Forgery (SSRF). To allow this URL, add it to " + "server_url_validation_allowed_base_urls." + ) + if allow_private_network_access: + return + blocked, category = try_categorize_non_public_address(address) if blocked: raise FunctionExecutionException( @@ -237,5 +329,7 @@ def _try_classify_ipv6(address: ipaddress.IPv6Address) -> tuple[bool, str]: return True, "multicast" if address in ipaddress.ip_network("2001:db8::/32"): return True, "reserved" + if address in _NAT64_LOCAL_USE_NETWORK: + return True, "NAT64 local-use prefix" return False, "" diff --git a/python/tests/unit/connectors/openapi_plugin/test_server_url_validator.py b/python/tests/unit/connectors/openapi_plugin/test_server_url_validator.py index 177ff15cb2c6..b7996a341722 100644 --- a/python/tests/unit/connectors/openapi_plugin/test_server_url_validator.py +++ b/python/tests/unit/connectors/openapi_plugin/test_server_url_validator.py @@ -1,8 +1,10 @@ # Copyright (c) Microsoft. All rights reserved. +import ipaddress import socket import pytest +from pytest import raises from semantic_kernel.connectors.openapi_plugin.server_url_validator import ( ServerUrlValidationOptions, @@ -17,7 +19,7 @@ [ ("127.0.0.1", "loopback"), ("127.255.255.254", "loopback"), - ("169.254.169.254", "link-local"), + ("169.254.10.10", "link-local"), ("169.254.0.1", "link-local"), ("10.0.0.1", "private (RFC1918)"), ("172.16.0.1", "private (RFC1918)"), @@ -76,7 +78,7 @@ def test_try_categorize_non_public_address_allows_public_addresses(address: str) async def test_validate_server_url_rejects_literal_link_local_ipv4(): with pytest.raises(FunctionExecutionException, match="link-local"): - await validate_server_url("https://169.254.169.254/latest/meta-data/") + await validate_server_url("https://169.254.10.10/latest/meta-data/") async def test_validate_server_url_rejects_literal_loopback_ipv6(): @@ -120,7 +122,7 @@ async def test_validate_server_url_allows_private_network_access_after_scheme_ga async def test_validate_server_url_blocks_hostname_resolving_to_link_local(): async def fake_resolver(host: str): assert host == "evil.example.com" - return ["169.254.169.254"] + return ["169.254.10.10"] with pytest.raises(FunctionExecutionException, match="link-local"): await validate_server_url("https://evil.example.com/latest/meta-data/", dns_resolver=fake_resolver) @@ -168,3 +170,108 @@ async def fake_resolver(host: str): with pytest.raises(FunctionExecutionException, match="returned no addresses"): await validate_server_url("https://empty-dns.example.com/", dns_resolver=fake_resolver) + + +CLOUD_METADATA_ENDPOINTS = [ + "169.254.169.254", # AWS IMDS, GCP, Azure, OCI, DigitalOcean + "169.254.170.2", # AWS ECS task IAM role credentials + "169.254.170.23", # AWS EKS Pod Identity Agent + "168.63.129.16", # Azure WireServer (publicly routable) + "100.100.100.200", # Alibaba Cloud + "192.0.0.192", # Oracle Cloud (Classic) + "169.254.42.42", # Scaleway +] + + +@pytest.mark.parametrize("address", CLOUD_METADATA_ENDPOINTS) +async def test_cloud_metadata_endpoints_blocked(address): + with raises(FunctionExecutionException): + await validate_server_url(f"https://{address}/latest/meta-data/") + + +@pytest.mark.parametrize("address", CLOUD_METADATA_ENDPOINTS) +async def test_cloud_metadata_endpoints_blocked_with_private_access(address): + """`allow_private_network_access` covers your own network, not the credential endpoint.""" + options = ServerUrlValidationOptions(allow_private_network_access=True) + with raises(FunctionExecutionException, match="cloud metadata endpoint"): + await validate_server_url(f"https://{address}/latest/meta-data/", options) + + +async def test_private_network_access_still_permits_rfc1918(): + options = ServerUrlValidationOptions(allow_private_network_access=True) + await validate_server_url("https://10.0.0.5/resource", options) + + +async def test_private_network_access_still_permits_loopback(): + options = ServerUrlValidationOptions(allow_private_network_access=True) + await validate_server_url("https://127.0.0.1/resource", options) + + +@pytest.mark.parametrize( + "address", + [ + "64:ff9b::169.254.169.254", # NAT64 well-known prefix (RFC 6052) + "64:ff9b:1::169.254.169.254", # NAT64 local-use prefix (RFC 8215) + "2002:a9fe:a9fe::", # 6to4 (RFC 3056) + "::ffff:169.254.169.254", # IPv4-mapped + ], +) +async def test_ipv6_forms_carrying_a_blocked_ipv4_are_rejected(address): + with raises(FunctionExecutionException): + await validate_server_url(f"https://[{address}]/latest/meta-data/") + + +async def test_teredo_carrying_a_blocked_ipv4_is_rejected(): + """Teredo obfuscates the client IPv4 by XOR-ing the low 32 bits with all ones.""" + target = ipaddress.IPv4Address("169.254.169.254") + low = bytes(byte ^ 0xFF for byte in target.packed) + teredo = ipaddress.IPv6Address(b"\x20\x01\x00\x00" + b"\x00" * 8 + low) + + with raises(FunctionExecutionException): + await validate_server_url(f"https://[{teredo}]/latest/meta-data/") + + +@pytest.mark.parametrize( + "address", + [ + "64:ff9b::1.1.1.1", # NAT64 well-known prefix pointing at a public IPv4 + "2002:0101:0101::", # 6to4 pointing at 1.1.1.1 + "2606:4700:4700::1111", # plain public IPv6 + ], +) +async def test_ipv6_forms_carrying_a_public_ipv4_are_allowed(address): + """Decoding must not reject legitimate NAT64/6to4 targets.""" + await validate_server_url(f"https://[{address}]/resource") + + +def _rfc6052(prefix_length: int, target: str) -> ipaddress.IPv6Address: + """Embed an IPv4 address in the RFC 8215 local-use prefix at one of RFC 6052's lengths.""" + packed = ipaddress.IPv4Address(target).packed + address = bytearray(16) + address[0:6] = b"\x00\x64\xff\x9b\x00\x01" + if prefix_length == 96: + address[12:16] = packed + elif prefix_length == 64: + address[9:13] = packed + elif prefix_length == 56: + address[7] = packed[0] + address[9:12] = packed[1:4] + elif prefix_length == 48: + address[6:8] = packed[0:2] + address[9:11] = packed[2:4] + return ipaddress.IPv6Address(bytes(address)) + + +@pytest.mark.parametrize("prefix_length", [96, 64, 56, 48]) +@pytest.mark.parametrize("target", ["1.1.1.1", "169.254.169.254"]) +async def test_nat64_local_use_prefix_is_blocked_at_every_embedding_length(prefix_length, target): + """The whole RFC 8215 local-use prefix is refused rather than decoded at a guessed length. + + RFC 6052 only puts the embedded address in bytes 12-15 for a /96; the shorter lengths put the + suffix there, which the RFC says SHOULD be zero. Decoding those bytes regardless read 0.0.0.0 + out of an ordinary public target and rejected it as "unspecified", while declining to decode + would have let the metadata address through at every length but /96. Neither is inferable from + the address, so the prefix is blocked as a whole. + """ + with raises(FunctionExecutionException): + await validate_server_url(f"https://[{_rfc6052(prefix_length, target)}]/resource")