diff --git a/AGENTS.md b/AGENTS.md index 7f84b48..11c4fee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,30 +203,30 @@ believing it works. --- -## 9. In flight +## 9. Recently landed -Region and service metadata capture is partially implemented on the branch -`feat/cloud-region-service`. All three providers publish it and the code currently discards -it at parse time: +Region and service metadata capture is **done**. All three providers publish it and the code +previously discarded it at parse time: -| Provider | Publishes | Historically kept | +| Provider | Publishes | Now captured as | |---|---|---| -| AWS | `ip_prefix`, `region`, `service`, `network_border_group` | prefix only | -| GCP | `ipv4Prefix`/`ipv6Prefix`, `service`, `scope` | prefix only | -| Azure | `addressPrefixes`, `region`, `systemService` | prefixes only | +| AWS | `ip_prefix`, `region`, `service` | region, service | +| GCP | `ipv4Prefix`/`ipv6Prefix`, `service`, `scope` | scope → region, service | +| Azure | `addressPrefixes`, `region`, `systemService` | region, systemService → service | -Without region and service a cloud match is not actionable downstream — the consumer needs -to know where to allocate and what the address belongs to. Of AWS's ~10,500 prefixes, over -half are the generic `AMAZON` tag, so "it is an AWS IP" is close to no information; `EC2` -in a named region is the useful signal. - -The work extends the fetchers to return `(ipv4, ipv6, metadata)` where metadata maps -CIDR → `(region, service)`, carries it on `CSPIPAddresses`, and writes one handoff record -per matched address: +Fetchers return `(ipv4, ipv6, metadata)` where metadata maps CIDR → `(region, service)`. +`CSPIPAddresses` carries it and exposes `describe(cidr)`. Each matched address is written as +one handoff record: ``` domain|ip|provider|region|service|prefix ``` -This changes the `csp_matches_*.txt` contract from prose to structured fields, and the -associated tests must be updated to assert the new contract rather than relaxed. +Why it mattered: over half of AWS's ~10,500 prefixes carry the generic `AMAZON` tag, so +"it is an AWS IP" was close to no information. A live run against AWS-fronted hosts now +reports `CLOUDFRONT` and `GLOBALACCELERATOR` rather than a flat provider tally — telling an +operator at a glance that these are CDN edges, not EC2 addresses in a region, and so not +worth pursuing. + +Fields are taken verbatim from the provider; where none is published they read `unknown` +rather than being inferred. The tool does not judge what is claimable — §1 applies. diff --git a/README.md b/README.md index 4c914af..0d7efdf 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Each run creates a timestamped subdirectory under the output directory containin | `resolution_results_*.txt` | Successfully resolved domains and their IPv4/IPv6 addresses, pipe-delimited (`domain\|ip1\|ip2`). Lines prefixed `WILDCARD\|` resolved only via a zone wildcard — see [Wildcard DNS detection](#wildcard-dns-detection) | | `unresolved_results_*.txt` | Domains that could not be resolved after all retries | | `takeover_candidates_*.txt` | Takeover candidates — `DANGLING\|` lines (dangling CNAMEs with category, recommendation, evidence) and `NS_TAKEOVER\|` lines (unresolvable nameservers) | -| `csp_matches_*.txt` | Domains resolving to cloud provider IP ranges (AWS, GCP, Azure — one line per match) | +| `csp_matches_*.txt` | One handoff record per matched address: `domain\|ip\|provider\|region\|service\|prefix`. Region and service come from the provider's own published ranges — see [Cloud IP attribution](#cloud-ip-attribution) | | `environment_results_*.json` | Run metadata (command, external IP, Docker status) | | `evidence/dns/` | dig or nslookup output per flagged domain (when `--evidence` is set) | @@ -117,6 +117,42 @@ lambda_handler.py — Lambda entry point → run(env_manager) Domain processing uses `asyncio.gather` with a `Semaphore` cap (`--max-threads`) to run many domains concurrently without exhausting file descriptors or triggering DNS rate limits. Failed domains are collected after each pass and retried up to `--retries` times. +## Cloud IP attribution + +Matching a resolved address to a cloud provider is only half an answer. `AWS` alone says +little: of roughly 10,500 published AWS prefixes, over half carry the generic `AMAZON` tag, +and the ones that matter operationally — `EC2` in a named region — look identical unless the +provider's own metadata is kept. + +Each match is therefore written as a record carrying the provider's published region and +service: + +``` +domain|ip|provider|region|service|prefix +``` + +``` +example.com|13.35.163.22|aws|GLOBAL|CLOUDFRONT|13.35.0.0/16 +example.com|3.11.53.7|aws|eu-west-2|EC2|3.8.0.0/14 +``` + +The difference is the point: the first is a CDN edge address, the second an EC2 address in a +specific region. Both are "AWS"; only one is a meaningful target. + +The end-of-run summary groups matches the same way: + +``` + CSP matches — AWS: 6 GCP: 0 Azure: 0 + by region and service: + 4 aws GLOBAL CLOUDFRONT + 2 aws GLOBAL GLOBALACCELERATOR +``` + +Metadata is taken verbatim from each provider (AWS `region`/`service`, GCP `scope`/`service`, +Azure `region`/`systemService`). Where a provider publishes none, the fields read `unknown` +rather than being inferred. DNSResolver does not judge which addresses are worth pursuing — +it reports what the provider states and leaves that decision to the operator. + ## Wildcard DNS detection A zone serving a wildcard record (`*.example.com`) answers for **every** name beneath it. Against an diff --git a/classes/csp_ip_addresses.py b/classes/csp_ip_addresses.py index f550d9a..2dc69da 100644 --- a/classes/csp_ip_addresses.py +++ b/classes/csp_ip_addresses.py @@ -1,11 +1,28 @@ class CSPIPAddresses: - def __init__(self, gcp_ipv4, gcp_ipv6, aws_ipv4, aws_ipv6, azure_ipv4, azure_ipv6): + def __init__( + self, + gcp_ipv4, + gcp_ipv6, + aws_ipv4, + aws_ipv6, + azure_ipv4, + azure_ipv6, + metadata=None, + ): self.gcp_ipv4 = gcp_ipv4 self.gcp_ipv6 = gcp_ipv6 self.aws_ipv4 = aws_ipv4 self.aws_ipv6 = aws_ipv6 self.azure_ipv4 = azure_ipv4 self.azure_ipv6 = azure_ipv6 + # CIDR -> (region, service). A match is only actionable downstream if the + # consumer knows where the address is allocated from and what it serves, + # so the publishers' own metadata is carried through rather than dropped. + self.metadata = metadata or {} + + def describe(self, cidr): + """Region and service for a matched prefix, or unknowns if unpublished.""" + return self.metadata.get(cidr, ("unknown", "unknown")) def get_gcp_ipv4(self): return self.gcp_ipv4 diff --git a/classes/run_summary.py b/classes/run_summary.py index 83b2a32..52a41cd 100644 --- a/classes/run_summary.py +++ b/classes/run_summary.py @@ -32,10 +32,21 @@ def display(self, total_input, failed_count): if ln.startswith("NS_TAKEOVER|") ] + # Handoff records: domain|ip|provider|region|service|prefix csp_lines = self._lines("csp") - aws_count = sum(1 for ln in csp_lines if "resolved to aws IPs" in ln) - gcp_count = sum(1 for ln in csp_lines if "resolved to gcp IPs" in ln) - azure_count = sum(1 for ln in csp_lines if "resolved to azure IPs" in ln) + csp_records = [ln.split("|") for ln in csp_lines if ln.count("|") >= 5] + aws_count = sum(1 for r in csp_records if r[2] == "aws") + gcp_count = sum(1 for r in csp_records if r[2] == "gcp") + azure_count = sum(1 for r in csp_records if r[2] == "azure") + + # Region and service are what an operator acts on, so surface the + # breakdown rather than a bare provider tally. + csp_breakdown = {} + for record in csp_records: + provider, region, service = record[2], record[3], record[4] + csp_breakdown[(provider, region, service)] = ( + csp_breakdown.get((provider, region, service), 0) + 1 + ) classified = {} unclassified = [] @@ -74,6 +85,12 @@ def display(self, total_input, failed_count): print( f" CSP matches — AWS: {aws_count} GCP: {gcp_count} Azure: {azure_count}" ) + if csp_breakdown: + print(" by region and service:") + for (provider, region, service), count in sorted( + csp_breakdown.items(), key=lambda kv: (-kv[1], kv[0]) + ): + print(f" {count:>4} {provider} {region} {service}") print(thin) if total_candidates == 0: diff --git a/imports/cloud_ip_ranges.py b/imports/cloud_ip_ranges.py index bf0b110..4b93fd2 100644 --- a/imports/cloud_ip_ranges.py +++ b/imports/cloud_ip_ranges.py @@ -4,7 +4,7 @@ import json import os import re -from typing import List, Tuple +from typing import Dict, List, Tuple from urllib.request import urlopen import requests @@ -13,82 +13,96 @@ IPV6_KEYWORDS = ["ipv6Prefix", "ipv6_prefix", "addressPrefixes"] -def fetch_ip_ranges_for_azure(url: str, extreme: bool) -> Tuple[List, List]: +def fetch_ip_ranges_for_azure(url: str, extreme: bool) -> Tuple[List, List, Dict]: try: response = requests.get(url, timeout=10) if response.status_code != 200: print( f"Failed to fetch IP ranges for Azure. Status code: {response.status_code}" ) - return [], [] + return [], [], {} data = json.loads(response.text) - ipv4_ranges = [ - item - for value in data.get("values", []) - for item in value.get("properties", {}).get("addressPrefixes", []) - if ":" not in item # Exclude IPv6 addresses - ] - ipv6_ranges = [ - item - for value in data.get("values", []) - for item in value.get("properties", {}).get("addressPrefixes", []) - if ":" in item # Only include IPv6 addresses - ] + ipv4_ranges = [] + ipv6_ranges = [] + metadata = {} + + for value in data.get("values", []): + props = value.get("properties", {}) + # Global tags carry an empty region; say so rather than inventing one. + region = props.get("region") or "global" + service = props.get("systemService") or value.get("name") or "unknown" + for item in props.get("addressPrefixes", []): + if ":" in item: + ipv6_ranges.append(item) + else: + ipv4_ranges.append(item) + metadata[item] = (region, service) + if extreme: print("IPv4 Ranges:", ipv4_ranges) print("IPv6 Ranges:", ipv6_ranges) - return ipv4_ranges, ipv6_ranges + return ipv4_ranges, ipv6_ranges, metadata except requests.exceptions.RequestException as e: print(f"An error occurred while fetching the IP ranges: {e}") - return [], [] + return [], [], {} -def fetch_ip_ranges(url: str, extreme: bool = False) -> Tuple[List, List]: +def fetch_ip_ranges(url: str, extreme: bool = False) -> Tuple[List, List, Dict]: try: response = requests.get(url, timeout=10) if response.status_code != 200: print(f"Failed to fetch IP ranges. Status code: {response.status_code}") - return [], [] + return [], [], {} data = json.loads(response.text) if "prefixes" not in data: print(f"No 'prefixes' key in retrieved data: {data}") - return [], [] - - ipv4_ranges = [ - prefix[keyword] - for prefix in data["prefixes"] - for keyword in IPV4_KEYWORDS - if keyword in prefix - ] - ipv6_ranges = [ - prefix[keyword] - for prefix in data["prefixes"] - for keyword in IPV6_KEYWORDS - if keyword in prefix - ] + return [], [], {} + + ipv4_ranges = [] + ipv6_ranges = [] + metadata = {} + + for prefix in data["prefixes"]: + # AWS calls it 'region', GCP calls it 'scope'. Both name the place an + # address is allocated from, which is what an operator needs in order + # to act on a match. + region = prefix.get("region") or prefix.get("scope") or "unknown" + service = prefix.get("service") or "unknown" + + for keyword in IPV4_KEYWORDS: + if keyword in prefix: + cidr = prefix[keyword] + ipv4_ranges.append(cidr) + metadata[cidr] = (region, service) + for keyword in IPV6_KEYWORDS: + if keyword in prefix: + cidr = prefix[keyword] + ipv6_ranges.append(cidr) + metadata[cidr] = (region, service) + if extreme: print("IPv4 Ranges:", ipv4_ranges) print("IPv6 Ranges:", ipv6_ranges) - return ipv4_ranges, ipv6_ranges + return ipv4_ranges, ipv6_ranges, metadata except requests.exceptions.RequestException as e: print(f"An error occurred while fetching the IP ranges: {e}") except IOError as e: print(f"An error occurred while writing to the file: {e}") - return [], [] + return [], [], {} def _fetch_and_save( url: str, filename: str, output_dir: str, extreme: bool -) -> Tuple[List, List]: +) -> Tuple[List, List, Dict]: ranges = fetch_ip_ranges(url, extreme) with open(os.path.join(output_dir, filename), "w", encoding="utf-8") as f: json.dump(ranges, f, indent=4) @@ -97,7 +111,7 @@ def _fetch_and_save( def fetch_google_cloud_ip_ranges( output_dir: str, extreme: bool = False -) -> Tuple[List, List]: +) -> Tuple[List, List, Dict]: return _fetch_and_save( "https://www.gstatic.com/ipranges/cloud.json", "gcp_ip_ranges.json", @@ -106,7 +120,9 @@ def fetch_google_cloud_ip_ranges( ) -def fetch_aws_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, List]: +def fetch_aws_ip_ranges( + output_dir: str, extreme: bool = False +) -> Tuple[List, List, Dict]: return _fetch_and_save( "https://ip-ranges.amazonaws.com/ip-ranges.json", "aws_ip_ranges.json", @@ -123,7 +139,7 @@ def fetch_aws_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, L AZURE_CACHE_PATH = ".azure_ip_cache.json" -def _save_azure_cache(ranges: Tuple[List, List]) -> None: +def _save_azure_cache(ranges: Tuple[List, List, Dict]) -> None: try: with open(AZURE_CACHE_PATH, "w", encoding="utf-8") as f: json.dump(ranges, f, indent=4) @@ -131,13 +147,18 @@ def _save_azure_cache(ranges: Tuple[List, List]) -> None: print(f"Warning: could not write Azure IP cache: {e}") -def _load_azure_cache() -> Tuple[List, List]: +def _load_azure_cache() -> Tuple[List, List, Dict]: with open(AZURE_CACHE_PATH, "r", encoding="utf-8") as f: data = json.load(f) - return data[0], data[1] + # Caches written before region/service capture hold only the two range lists. + # Read them rather than discarding a usable cache; metadata is simply absent. + metadata = data[2] if len(data) > 2 else {} + return data[0], data[1], metadata -def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, List]: +def fetch_azure_ip_ranges( + output_dir: str, extreme: bool = False +) -> Tuple[List, List, Dict]: confirmation_url = ( "https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519" ) @@ -160,7 +181,7 @@ def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, # Step 2: if scrape succeeded, fetch and return if json_url: ranges = fetch_ip_ranges_for_azure(json_url, extreme) - if ranges != ([], []): + if ranges[0] or ranges[1]: _save_azure_cache(ranges) with open( os.path.join(output_dir, "azure_ip_ranges.json"), "w", encoding="utf-8" @@ -185,7 +206,7 @@ def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, "This may be stale — update AZURE_PINNED_URL in cloud_ip_ranges.py if needed." ) ranges = fetch_ip_ranges_for_azure(AZURE_PINNED_URL, extreme) - if ranges != ([], []): + if ranges[0] or ranges[1]: _save_azure_cache(ranges) with open( os.path.join(output_dir, "azure_ip_ranges.json"), "w", encoding="utf-8" @@ -194,4 +215,4 @@ def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, return ranges print("Azure IP fetch: all sources exhausted — no Azure ranges loaded.") - return [], [] + return [], [], {} diff --git a/imports/cloud_service_provider_checks.py b/imports/cloud_service_provider_checks.py index 763e6b4..a671ab9 100644 --- a/imports/cloud_service_provider_checks.py +++ b/imports/cloud_service_provider_checks.py @@ -75,7 +75,7 @@ def get_vendor_ips(domain_context, ip_version): def get_ip_matches(final_ips, vendor_ips_context, domain_context, ip_version): - matches = {vendor: set() for vendor in vendor_ips_context} + matches = {vendor: {} for vendor in vendor_ips_context} for ip in final_ips: if not is_ip_version(ip, ip_version): continue @@ -102,12 +102,14 @@ def match_ip_with_vendors(ip_obj, vendor_ips_context, domain_context, matches): domain_context.log_info( f"IP {ip_obj} is in range {ip_range} for vendor {vendor}" ) - matches[vendor].add(str(ip_obj)) + # Keep the prefix that matched — it is the key to the region and + # service the provider published for it. + matches[vendor][str(ip_obj)] = ip_range def merge_matches(matches_ipv4, matches_ipv6, vendor_ips_context): return { - vendor: list(matches_ipv4[vendor] | matches_ipv6[vendor]) + vendor: {**matches_ipv4[vendor], **matches_ipv6[vendor]} for vendor in vendor_ips_context } @@ -115,18 +117,33 @@ def merge_matches(matches_ipv4, matches_ipv6, vendor_ips_context): def log_and_write( vendor, matched_ips, domain, output_files, domain_context, written_lines ): - message = f"{domain} resolved to {vendor} IPs: {matched_ips}" + """ + Write one line per matched address, as a handoff record for downstream + tooling: domain|ip|provider|region|service|prefix + + One address per line, pipe-delimited, because this file is consumed by + another tool rather than read as prose. Region and service come from the + provider's own published ranges and are what make a match actionable — an + address is only worth pursuing if you know where it is allocated from and + what it belongs to. + """ + csp_ip_addresses = domain_context.get_csp_ip_addresses() + file_path = output_files["standard"]["csp"] + wrote_any = False - # Deduplicate against an in-memory, run-scoped set of lines already - # written — avoids re-reading the whole output file on every call. - if message in written_lines: - return False + for ip, prefix in sorted(matched_ips.items()): + region, service = csp_ip_addresses.describe(prefix) + message = f"{domain}|{ip}|{vendor}|{region}|{service}|{prefix}" - file_path = output_files["standard"]["csp"] - with open(file_path, "a", encoding="utf-8") as file: - file.write(message + "\n") - written_lines.add(message) + # Deduplicate against an in-memory, run-scoped set of lines already + # written — avoids re-reading the whole output file on every call. + if message in written_lines: + continue - domain_context.log_info(message) + with open(file_path, "a", encoding="utf-8") as file: + file.write(message + "\n") + written_lines.add(message) + domain_context.log_info(message) + wrote_any = True - return True + return wrote_any diff --git a/resolver.py b/resolver.py index c05483b..f66b85c 100644 --- a/resolver.py +++ b/resolver.py @@ -21,18 +21,24 @@ async def run(env_manager): both the CLI entrypoint (resolver.py) and the Lambda entrypoint (lambda_handler.py) can share the same logic. """ - gcp_ipv4, gcp_ipv6 = fetch_google_cloud_ip_ranges( + gcp_ipv4, gcp_ipv6, gcp_meta = fetch_google_cloud_ip_ranges( env_manager.output_dir, env_manager.extreme ) - aws_ipv4, aws_ipv6 = fetch_aws_ip_ranges( + aws_ipv4, aws_ipv6, aws_meta = fetch_aws_ip_ranges( env_manager.output_dir, env_manager.extreme ) - azure_ipv4, azure_ipv6 = fetch_azure_ip_ranges( + azure_ipv4, azure_ipv6, azure_meta = fetch_azure_ip_ranges( env_manager.output_dir, env_manager.extreme ) csp_ip_addresses = CSPIPAddresses( - gcp_ipv4, gcp_ipv6, aws_ipv4, aws_ipv6, azure_ipv4, azure_ipv6 + gcp_ipv4, + gcp_ipv6, + aws_ipv4, + aws_ipv6, + azure_ipv4, + azure_ipv6, + metadata={**gcp_meta, **aws_meta, **azure_meta}, ) env_manager.set_domains() diff --git a/tests/conftest.py b/tests/conftest.py index 77bb8c6..871cb97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,12 +23,33 @@ AZURE_IPV4 = ["13.0.0.0/8", "20.0.0.0/8"] AZURE_IPV6 = ["2603:1000::/24"] +# CIDR -> (region, service), as the providers publish it. A match is only +# actionable downstream if the consumer knows where the address is allocated +# from and what it belongs to. +CSP_METADATA = { + "34.0.0.0/8": ("europe-west2", "Google Cloud"), + "35.0.0.0/8": ("us-central1", "Google Cloud"), + "2600:1900::/35": ("europe-west2", "Google Cloud"), + "3.0.0.0/8": ("eu-west-2", "EC2"), + "52.0.0.0/8": ("us-east-1", "AMAZON"), + "2600:1f00::/25": ("eu-west-2", "EC2"), + "13.0.0.0/8": ("uksouth", "AzureCloud"), + "20.0.0.0/8": ("global", "AzureCloud"), + "2603:1000::/24": ("uksouth", "AzureCloud"), +} + @pytest.fixture def csp_ips(): """A real CSPIPAddresses instance populated with test ranges.""" return CSPIPAddresses( - GCP_IPV4, GCP_IPV6, AWS_IPV4, AWS_IPV6, AZURE_IPV4, AZURE_IPV6 + GCP_IPV4, + GCP_IPV6, + AWS_IPV4, + AWS_IPV6, + AZURE_IPV4, + AZURE_IPV6, + metadata=CSP_METADATA, ) diff --git a/tests/test_cloud_ip_ranges.py b/tests/test_cloud_ip_ranges.py index 4f2ef93..1f23fb7 100644 --- a/tests/test_cloud_ip_ranges.py +++ b/tests/test_cloud_ip_ranges.py @@ -43,7 +43,7 @@ def test_fetch_ip_ranges_returns_ipv4_and_ipv6(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=GCP_PAYLOAD), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert "34.0.0.0/8" in v4 assert "2600:1900::/28" in v6 @@ -54,7 +54,7 @@ def test_fetch_ip_ranges_non_200_returns_empty(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(status_code=503), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert v4 == [] assert v6 == [] @@ -65,7 +65,7 @@ def test_fetch_ip_ranges_missing_prefixes_key_returns_empty(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body={"other": []}), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert v4 == [] assert v6 == [] @@ -76,7 +76,7 @@ def test_fetch_ip_ranges_request_exception_returns_empty(): "imports.cloud_ip_ranges.requests.get", side_effect=requests.exceptions.RequestException("timeout"), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert v4 == [] assert v6 == [] @@ -87,7 +87,7 @@ def test_fetch_ip_ranges_extreme_flag_does_not_raise(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=GCP_PAYLOAD), ): - v4, v6 = fetch_ip_ranges("http://fake-url", extreme=True) + v4, v6, _meta = fetch_ip_ranges("http://fake-url", extreme=True) assert "34.0.0.0/8" in v4 @@ -106,7 +106,9 @@ def test_fetch_ip_ranges_for_azure_splits_ipv4_and_ipv6(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=AZURE_PAYLOAD), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=False) + v4, v6, _meta = fetch_ip_ranges_for_azure( + "http://fake-azure-url", extreme=False + ) assert "13.64.0.0/11" in v4 assert "2603:1010::/48" in v6 @@ -117,7 +119,9 @@ def test_fetch_ip_ranges_for_azure_non_200_returns_empty(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(status_code=404), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=False) + v4, v6, _meta = fetch_ip_ranges_for_azure( + "http://fake-azure-url", extreme=False + ) assert v4 == [] assert v6 == [] @@ -128,7 +132,9 @@ def test_fetch_ip_ranges_for_azure_request_exception_returns_empty(): "imports.cloud_ip_ranges.requests.get", side_effect=requests.exceptions.RequestException("timeout"), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=False) + v4, v6, _meta = fetch_ip_ranges_for_azure( + "http://fake-azure-url", extreme=False + ) assert v4 == [] assert v6 == [] @@ -139,7 +145,7 @@ def test_fetch_ip_ranges_for_azure_extreme_does_not_raise(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=AZURE_PAYLOAD), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=True) + v4, v6, _meta = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=True) assert "13.64.0.0/11" in v4 @@ -246,4 +252,122 @@ def test_fetch_azure_ip_ranges_all_sources_fail_returns_empty(tmp_path): ): result = fetch_azure_ip_ranges(str(tmp_path)) - assert result == ([], []) + assert result[:2] == ([], []) + + +# --------------------------------------------------------------------------- +# Region / service metadata capture +# +# A cloud match is only actionable downstream if the consumer knows where the +# address is allocated from and what it belongs to. All three providers publish +# that alongside the prefix, so it must survive parsing. +# --------------------------------------------------------------------------- + + +def test_aws_region_and_service_are_captured(monkeypatch): + """AWS publishes region and service per prefix; both must be retained.""" + payload = { + "prefixes": [ + {"ip_prefix": "3.5.140.0/22", "region": "eu-west-2", "service": "EC2"}, + {"ip_prefix": "52.94.0.0/22", "region": "us-east-1", "service": "AMAZON"}, + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/aws.json") + + assert "3.5.140.0/22" in ipv4 + assert meta["3.5.140.0/22"] == ("eu-west-2", "EC2") + assert meta["52.94.0.0/22"] == ("us-east-1", "AMAZON") + + +def test_gcp_scope_is_captured_as_region(monkeypatch): + """GCP calls it 'scope'; it means the same thing and must map through.""" + payload = { + "prefixes": [ + { + "ipv4Prefix": "34.1.208.0/20", + "service": "Google Cloud", + "scope": "europe-west2", + } + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + _ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/gcp.json") + + assert meta["34.1.208.0/20"] == ("europe-west2", "Google Cloud") + + +def test_missing_metadata_is_reported_as_unknown_not_invented(monkeypatch): + """A prefix without region/service must say so rather than guess.""" + payload = {"prefixes": [{"ip_prefix": "198.51.100.0/24"}]} + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + _ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/x.json") + + assert meta["198.51.100.0/24"] == ("unknown", "unknown") + + +def test_azure_region_and_system_service_are_captured(monkeypatch): + """Azure carries region and systemService on the tag's properties.""" + payload = { + "values": [ + { + "name": "AzureCloud.uksouth", + "properties": { + "region": "uksouth", + "systemService": "AzureCloud", + "addressPrefixes": ["20.26.0.0/16", "2603:1000::/24"], + }, + } + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + ipv4, ipv6, meta = fetch_ip_ranges_for_azure( + "http://example.invalid/az.json", False + ) + + assert ipv4 == ["20.26.0.0/16"] + assert ipv6 == ["2603:1000::/24"] + assert meta["20.26.0.0/16"] == ("uksouth", "AzureCloud") + assert meta["2603:1000::/24"] == ("uksouth", "AzureCloud") + + +def test_azure_global_tag_region_is_labelled_global(monkeypatch): + """Azure leaves region empty on global tags; that must not read as unknown.""" + payload = { + "values": [ + { + "name": "AzureCloud", + "properties": { + "region": "", + "systemService": "", + "addressPrefixes": ["13.64.0.0/16"], + }, + } + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + _ipv4, _ipv6, meta = fetch_ip_ranges_for_azure( + "http://example.invalid/az.json", False + ) + + assert meta["13.64.0.0/16"] == ("global", "AzureCloud") diff --git a/tests/test_cloud_service_provider_checks.py b/tests/test_cloud_service_provider_checks.py index 825b39e..92e412a 100644 --- a/tests/test_cloud_service_provider_checks.py +++ b/tests/test_cloud_service_provider_checks.py @@ -86,33 +86,35 @@ def test_get_vendor_ips_unknown_version_returns_empty(ctx): def test_match_ip_with_vendors_records_match(ctx): ip_obj = ipaddress.IPv4Address("34.1.2.3") vendor_ips = {"gcp": ["34.0.0.0/8"], "aws": [], "azure": []} - matches = {"gcp": set(), "aws": set(), "azure": set()} + matches = {"gcp": {}, "aws": {}, "azure": {}} match_ip_with_vendors(ip_obj, vendor_ips, ctx, matches) assert "34.1.2.3" in matches["gcp"] - assert matches["aws"] == set() + # The matched prefix is retained — it is the key to region and service. + assert matches["gcp"]["34.1.2.3"] == "34.0.0.0/8" + assert matches["aws"] == {} def test_match_ip_with_vendors_no_match(ctx): ip_obj = ipaddress.IPv4Address("1.2.3.4") vendor_ips = {"gcp": ["34.0.0.0/8"]} - matches = {"gcp": set()} + matches = {"gcp": {}} match_ip_with_vendors(ip_obj, vendor_ips, ctx, matches) - assert matches["gcp"] == set() + assert matches["gcp"] == {} def test_match_ip_with_vendors_ignores_invalid_range(ctx): """A bad CIDR string should not crash — it's logged and skipped.""" ip_obj = ipaddress.IPv4Address("1.2.3.4") vendor_ips = {"gcp": ["not-a-valid-cidr"]} - matches = {"gcp": set()} + matches = {"gcp": {}} match_ip_with_vendors(ip_obj, vendor_ips, ctx, matches) - assert matches["gcp"] == set() + assert matches["gcp"] == {} # --------------------------------------------------------------------------- @@ -121,21 +123,22 @@ def test_match_ip_with_vendors_ignores_invalid_range(ctx): def test_merge_matches_combines_ipv4_and_ipv6(): - v4 = {"gcp": {"1.2.3.4"}, "aws": set()} - v6 = {"gcp": {"::1"}, "aws": set()} + """Both families merge, each address keeping the prefix it matched.""" + v4 = {"gcp": {"1.2.3.4": "1.2.0.0/16"}, "aws": {}} + v6 = {"gcp": {"::1": "::/64"}, "aws": {}} vendor_context = {"gcp": [], "aws": []} result = merge_matches(v4, v6, vendor_context) - assert set(result["gcp"]) == {"1.2.3.4", "::1"} - assert result["aws"] == [] + assert result["gcp"] == {"1.2.3.4": "1.2.0.0/16", "::1": "::/64"} + assert result["aws"] == {} def test_merge_matches_empty_sets(): - v4 = {"gcp": set()} - v6 = {"gcp": set()} + v4 = {"gcp": {}} + v6 = {"gcp": {}} result = merge_matches(v4, v6, {"gcp": []}) - assert result["gcp"] == [] + assert result["gcp"] == {} # --------------------------------------------------------------------------- @@ -146,13 +149,13 @@ def test_merge_matches_empty_sets(): def test_get_ip_matches_finds_gcp_ipv4(ctx, csp_ips): # 34.1.2.3 falls in GCP's 34.0.0.0/8 range (from conftest) result = get_ip_matches(["34.1.2.3"], get_vendor_ips(ctx, 4), ctx, ip_version=4) - assert "34.1.2.3" in result["gcp"] + assert result["gcp"]["34.1.2.3"] == "34.0.0.0/8" def test_get_ip_matches_skips_wrong_ip_version(ctx): # An IPv6 address should be skipped when looking for IPv4 matches result = get_ip_matches(["2600:1900::1"], get_vendor_ips(ctx, 4), ctx, ip_version=4) - assert all(len(s) == 0 for s in result.values()) + assert all(len(m) == 0 for m in result.values()) # --------------------------------------------------------------------------- @@ -165,10 +168,17 @@ def test_log_and_write_creates_entry(tmp_path, ctx): out_file.touch() output_files = {"standard": {"csp": str(out_file)}} - result = log_and_write("gcp", ["34.1.2.3"], "example.com", output_files, ctx, set()) + result = log_and_write( + "gcp", {"34.1.2.3": "34.0.0.0/8"}, "example.com", output_files, ctx, set() + ) assert result is True - assert "example.com resolved to gcp IPs" in out_file.read_text() + # One handoff record per address: domain|ip|provider|region|service|prefix. + # Region and service are what make the match actionable downstream. + assert ( + out_file.read_text().strip() + == "example.com|34.1.2.3|gcp|europe-west2|Google Cloud|34.0.0.0/8" + ) def test_log_and_write_no_duplicate_entries(tmp_path, ctx): @@ -179,10 +189,20 @@ def test_log_and_write_no_duplicate_entries(tmp_path, ctx): written_lines = set() first = log_and_write( - "gcp", ["34.1.2.3"], "example.com", output_files, ctx, written_lines + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, ) second = log_and_write( - "gcp", ["34.1.2.3"], "example.com", output_files, ctx, written_lines + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, ) assert first is True @@ -207,9 +227,16 @@ def guarded_open(file, mode="r", *args, **kwargs): monkeypatch.setattr("builtins.open", guarded_open) - log_and_write("gcp", ["34.1.2.3"], "example.com", output_files, ctx, written_lines) + log_and_write( + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, + ) - assert "example.com resolved to gcp IPs" in out_file.read_text() + assert "example.com|34.1.2.3|gcp|" in out_file.read_text() def test_log_and_write_substring_line_not_suppressed(tmp_path, ctx): @@ -221,14 +248,18 @@ def test_log_and_write_substring_line_not_suppressed(tmp_path, ctx): output_files = {"standard": {"csp": str(out_file)}} written_lines = set() - # "example.com resolved to gcp IPs: ['1.1.1.1']" contains - # "ple.com resolved to gcp IPs: ['1.1.1.1']" as a substring + # "example.com|34.1.2.3|..." contains "ple.com|34.1.2.3|..." as a substring # (example.com = exam + ple.com), but they are different domains/lines. first = log_and_write( - "gcp", ["1.1.1.1"], "example.com", output_files, ctx, written_lines + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, ) second = log_and_write( - "gcp", ["1.1.1.1"], "ple.com", output_files, ctx, written_lines + "gcp", {"34.1.2.3": "34.0.0.0/8"}, "ple.com", output_files, ctx, written_lines ) assert first is True @@ -316,11 +347,11 @@ def test_parse_network_equivalence_ip_inside_and_outside_range(ctx): inside_ip = ipaddress.IPv4Address("34.1.2.3") outside_ip = ipaddress.IPv4Address("8.8.8.8") - matches = {"gcp": set(), "aws": set(), "azure": set()} + matches = {"gcp": {}, "aws": {}, "azure": {}} match_ip_with_vendors(inside_ip, vendor_ips, ctx, matches) match_ip_with_vendors(outside_ip, vendor_ips, ctx, matches) - assert matches == {"gcp": {"34.1.2.3"}, "aws": set(), "azure": set()} + assert matches == {"gcp": {"34.1.2.3": "34.0.0.0/8"}, "aws": {}, "azure": {}} def test_parse_network_is_memoised_across_calls(): @@ -348,7 +379,7 @@ def test_match_ip_with_vendors_reuses_cached_network(ctx): re-parse the CIDR strings for each IP.""" parse_network.cache_clear() vendor_ips = {"gcp": ["34.0.0.0/8", "35.0.0.0/8"], "aws": [], "azure": []} - matches = {"gcp": set(), "aws": set(), "azure": set()} + matches = {"gcp": {}, "aws": {}, "azure": {}} ips = [ipaddress.IPv4Address(f"34.1.2.{i}") for i in range(10)] for ip in ips: @@ -359,3 +390,44 @@ def test_match_ip_with_vendors_reuses_cached_network(ctx): # regardless of how many IPs (10) were checked against them. assert info.misses == 2 assert info.hits == 10 * 2 - 2 + + +def test_handoff_record_carries_region_and_service_end_to_end( + tmp_path, mock_env_manager, ctx +): + """ + The whole point of the CSP output: a downstream tool must be able to read + where an address is allocated from and what it belongs to, without going + back to the provider's range files. + """ + out_file = tmp_path / "csp.txt" + out_file.touch() + mock_env_manager.output_files = {"standard": {"csp": str(out_file)}} + + # 3.1.2.3 falls in AWS 3.0.0.0/8, published as eu-west-2 / EC2 in conftest. + perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3"]) + + line = out_file.read_text().strip() + assert line == "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8" + + domain, ip, provider, region, service, prefix = line.split("|") + assert (provider, region, service) == ("aws", "eu-west-2", "EC2") + + +def test_each_matched_address_gets_its_own_record(tmp_path, mock_env_manager, ctx): + """ + One line per address, not one line per domain with a list. Addresses in the + same provider can sit in different regions, and a consumer acts per address. + """ + out_file = tmp_path / "csp.txt" + out_file.touch() + mock_env_manager.output_files = {"standard": {"csp": str(out_file)}} + + # 3.x -> eu-west-2/EC2, 52.x -> us-east-1/AMAZON (both AWS, different regions) + perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3", "52.1.2.3"]) + + lines = sorted(ln for ln in out_file.read_text().splitlines() if ln) + assert lines == [ + "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8", + "example.com|52.1.2.3|aws|us-east-1|AMAZON|52.0.0.0/8", + ] diff --git a/tests/test_run_summary.py b/tests/test_run_summary.py index 2b0fee3..149d465 100644 --- a/tests/test_run_summary.py +++ b/tests/test_run_summary.py @@ -45,9 +45,9 @@ def test_counts_are_accurate(tmp_path, capsys): "resolved": "a.com|1.1.1.1\nb.com|2.2.2.2\n", "unresolved": "DNS resolution error for c.com: timeout\n", "csp": ( - "a.com resolved to aws IPs: ['1.1.1.1']\n" - "b.com resolved to gcp IPs: ['2.2.2.2']\n" - "b2.com resolved to gcp IPs: ['3.3.3.3']\n" + "a.com|1.1.1.1|aws|eu-west-2|EC2|1.1.0.0/16\n" + "b.com|2.2.2.2|gcp|europe-west2|Google Cloud|2.2.0.0/16\n" + "b2.com|3.3.3.3|gcp|us-central1|Google Cloud|3.3.0.0/16\n" ), }, ) @@ -212,3 +212,28 @@ def test_missing_file_treated_as_empty(tmp_path, capsys): out = capsys.readouterr().out assert "No takeover candidates detected." in out + + +def test_csp_breakdown_shows_region_and_service(tmp_path, capsys): + """ + A provider tally alone is not actionable — an operator needs the region an + address is allocated from and the service it belongs to. + """ + summary = _make_summary( + tmp_path, + { + "csp": ( + "a.com|1.1.1.1|aws|eu-west-2|EC2|1.1.0.0/16\n" + "b.com|1.1.1.2|aws|eu-west-2|EC2|1.1.0.0/16\n" + "c.com|9.9.9.9|aws|us-east-1|AMAZON|9.9.0.0/16\n" + ), + }, + ) + summary.display(total_input=3, failed_count=0) + + out = capsys.readouterr().out + assert "by region and service" in out + assert "aws eu-west-2 EC2" in out + assert "aws us-east-1 AMAZON" in out + # The busiest grouping is listed first. + assert out.index("eu-west-2") < out.index("us-east-1")