From feae5d048110a82c3fcb54d8123347c434b87929 Mon Sep 17 00:00:00 2001 From: incendiary Date: Fri, 31 Jul 2026 01:13:15 +0800 Subject: [PATCH] feat: capture cloud region and service, emit structured handoff records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matching an address to a provider was only half an answer. "It is an AWS IP" is close to no information: of roughly 10,500 published AWS prefixes, over half carry the generic AMAZON tag, and the ones that matter operationally — EC2 in a named region — were indistinguishable from CDN edge addresses. All three providers publish region and service alongside each prefix, and all of it was discarded at parse time. AWS gives region and service, GCP gives scope and service, Azure gives region and systemService on the tag. The fetchers now return that metadata, CSPIPAddresses carries it, and each matched address is written as its own record: domain|ip|provider|region|service|prefix The CSP output is consumed by other tooling rather than read as prose, so it is now structured fields rather than a sentence, one line per address rather than a list per domain. Addresses under one provider can sit in different regions, and a consumer acts per address. The run summary groups matches by provider, region and service for the same reason. A live run against AWS-fronted hosts reports CLOUDFRONT and GLOBALACCELERATOR, which tells an operator these are CDN edges rather than EC2 addresses in a region — a distinction the previous output could not make. Fields are taken verbatim from the provider. Where none is published they read "unknown" rather than being inferred, and no judgement is encoded about what is worth pursuing; that is the operator's call. Tests changed expectation where they asserted the old prose format, and new tests cover metadata capture for each provider, the unknown fallback, Azure's empty region on global tags, and the end-to-end handoff record. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 36 ++--- README.md | 38 +++++- classes/csp_ip_addresses.py | 19 ++- classes/run_summary.py | 23 +++- imports/cloud_ip_ranges.py | 111 +++++++++------ imports/cloud_service_provider_checks.py | 45 ++++-- resolver.py | 14 +- tests/conftest.py | 23 +++- tests/test_cloud_ip_ranges.py | 144 ++++++++++++++++++-- tests/test_cloud_service_provider_checks.py | 128 +++++++++++++---- tests/test_run_summary.py | 31 ++++- 11 files changed, 484 insertions(+), 128 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7f84b48..11c4fee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,30 +203,30 @@ believing it works. --- -## 9. In flight +## 9. Recently landed -Region and service metadata capture is partially implemented on the branch -`feat/cloud-region-service`. All three providers publish it and the code currently discards -it at parse time: +Region and service metadata capture is **done**. All three providers publish it and the code +previously discarded it at parse time: -| Provider | Publishes | Historically kept | +| Provider | Publishes | Now captured as | |---|---|---| -| AWS | `ip_prefix`, `region`, `service`, `network_border_group` | prefix only | -| GCP | `ipv4Prefix`/`ipv6Prefix`, `service`, `scope` | prefix only | -| Azure | `addressPrefixes`, `region`, `systemService` | prefixes only | +| AWS | `ip_prefix`, `region`, `service` | region, service | +| GCP | `ipv4Prefix`/`ipv6Prefix`, `service`, `scope` | scope → region, service | +| Azure | `addressPrefixes`, `region`, `systemService` | region, systemService → service | -Without region and service a cloud match is not actionable downstream — the consumer needs -to know where to allocate and what the address belongs to. Of AWS's ~10,500 prefixes, over -half are the generic `AMAZON` tag, so "it is an AWS IP" is close to no information; `EC2` -in a named region is the useful signal. - -The work extends the fetchers to return `(ipv4, ipv6, metadata)` where metadata maps -CIDR → `(region, service)`, carries it on `CSPIPAddresses`, and writes one handoff record -per matched address: +Fetchers return `(ipv4, ipv6, metadata)` where metadata maps CIDR → `(region, service)`. +`CSPIPAddresses` carries it and exposes `describe(cidr)`. Each matched address is written as +one handoff record: ``` domain|ip|provider|region|service|prefix ``` -This changes the `csp_matches_*.txt` contract from prose to structured fields, and the -associated tests must be updated to assert the new contract rather than relaxed. +Why it mattered: over half of AWS's ~10,500 prefixes carry the generic `AMAZON` tag, so +"it is an AWS IP" was close to no information. A live run against AWS-fronted hosts now +reports `CLOUDFRONT` and `GLOBALACCELERATOR` rather than a flat provider tally — telling an +operator at a glance that these are CDN edges, not EC2 addresses in a region, and so not +worth pursuing. + +Fields are taken verbatim from the provider; where none is published they read `unknown` +rather than being inferred. The tool does not judge what is claimable — §1 applies. diff --git a/README.md b/README.md index 4c914af..0d7efdf 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Each run creates a timestamped subdirectory under the output directory containin | `resolution_results_*.txt` | Successfully resolved domains and their IPv4/IPv6 addresses, pipe-delimited (`domain\|ip1\|ip2`). Lines prefixed `WILDCARD\|` resolved only via a zone wildcard — see [Wildcard DNS detection](#wildcard-dns-detection) | | `unresolved_results_*.txt` | Domains that could not be resolved after all retries | | `takeover_candidates_*.txt` | Takeover candidates — `DANGLING\|` lines (dangling CNAMEs with category, recommendation, evidence) and `NS_TAKEOVER\|` lines (unresolvable nameservers) | -| `csp_matches_*.txt` | Domains resolving to cloud provider IP ranges (AWS, GCP, Azure — one line per match) | +| `csp_matches_*.txt` | One handoff record per matched address: `domain\|ip\|provider\|region\|service\|prefix`. Region and service come from the provider's own published ranges — see [Cloud IP attribution](#cloud-ip-attribution) | | `environment_results_*.json` | Run metadata (command, external IP, Docker status) | | `evidence/dns/` | dig or nslookup output per flagged domain (when `--evidence` is set) | @@ -117,6 +117,42 @@ lambda_handler.py — Lambda entry point → run(env_manager) Domain processing uses `asyncio.gather` with a `Semaphore` cap (`--max-threads`) to run many domains concurrently without exhausting file descriptors or triggering DNS rate limits. Failed domains are collected after each pass and retried up to `--retries` times. +## Cloud IP attribution + +Matching a resolved address to a cloud provider is only half an answer. `AWS` alone says +little: of roughly 10,500 published AWS prefixes, over half carry the generic `AMAZON` tag, +and the ones that matter operationally — `EC2` in a named region — look identical unless the +provider's own metadata is kept. + +Each match is therefore written as a record carrying the provider's published region and +service: + +``` +domain|ip|provider|region|service|prefix +``` + +``` +example.com|13.35.163.22|aws|GLOBAL|CLOUDFRONT|13.35.0.0/16 +example.com|3.11.53.7|aws|eu-west-2|EC2|3.8.0.0/14 +``` + +The difference is the point: the first is a CDN edge address, the second an EC2 address in a +specific region. Both are "AWS"; only one is a meaningful target. + +The end-of-run summary groups matches the same way: + +``` + CSP matches — AWS: 6 GCP: 0 Azure: 0 + by region and service: + 4 aws GLOBAL CLOUDFRONT + 2 aws GLOBAL GLOBALACCELERATOR +``` + +Metadata is taken verbatim from each provider (AWS `region`/`service`, GCP `scope`/`service`, +Azure `region`/`systemService`). Where a provider publishes none, the fields read `unknown` +rather than being inferred. DNSResolver does not judge which addresses are worth pursuing — +it reports what the provider states and leaves that decision to the operator. + ## Wildcard DNS detection A zone serving a wildcard record (`*.example.com`) answers for **every** name beneath it. Against an diff --git a/classes/csp_ip_addresses.py b/classes/csp_ip_addresses.py index f550d9a..2dc69da 100644 --- a/classes/csp_ip_addresses.py +++ b/classes/csp_ip_addresses.py @@ -1,11 +1,28 @@ class CSPIPAddresses: - def __init__(self, gcp_ipv4, gcp_ipv6, aws_ipv4, aws_ipv6, azure_ipv4, azure_ipv6): + def __init__( + self, + gcp_ipv4, + gcp_ipv6, + aws_ipv4, + aws_ipv6, + azure_ipv4, + azure_ipv6, + metadata=None, + ): self.gcp_ipv4 = gcp_ipv4 self.gcp_ipv6 = gcp_ipv6 self.aws_ipv4 = aws_ipv4 self.aws_ipv6 = aws_ipv6 self.azure_ipv4 = azure_ipv4 self.azure_ipv6 = azure_ipv6 + # CIDR -> (region, service). A match is only actionable downstream if the + # consumer knows where the address is allocated from and what it serves, + # so the publishers' own metadata is carried through rather than dropped. + self.metadata = metadata or {} + + def describe(self, cidr): + """Region and service for a matched prefix, or unknowns if unpublished.""" + return self.metadata.get(cidr, ("unknown", "unknown")) def get_gcp_ipv4(self): return self.gcp_ipv4 diff --git a/classes/run_summary.py b/classes/run_summary.py index 83b2a32..52a41cd 100644 --- a/classes/run_summary.py +++ b/classes/run_summary.py @@ -32,10 +32,21 @@ def display(self, total_input, failed_count): if ln.startswith("NS_TAKEOVER|") ] + # Handoff records: domain|ip|provider|region|service|prefix csp_lines = self._lines("csp") - aws_count = sum(1 for ln in csp_lines if "resolved to aws IPs" in ln) - gcp_count = sum(1 for ln in csp_lines if "resolved to gcp IPs" in ln) - azure_count = sum(1 for ln in csp_lines if "resolved to azure IPs" in ln) + csp_records = [ln.split("|") for ln in csp_lines if ln.count("|") >= 5] + aws_count = sum(1 for r in csp_records if r[2] == "aws") + gcp_count = sum(1 for r in csp_records if r[2] == "gcp") + azure_count = sum(1 for r in csp_records if r[2] == "azure") + + # Region and service are what an operator acts on, so surface the + # breakdown rather than a bare provider tally. + csp_breakdown = {} + for record in csp_records: + provider, region, service = record[2], record[3], record[4] + csp_breakdown[(provider, region, service)] = ( + csp_breakdown.get((provider, region, service), 0) + 1 + ) classified = {} unclassified = [] @@ -74,6 +85,12 @@ def display(self, total_input, failed_count): print( f" CSP matches — AWS: {aws_count} GCP: {gcp_count} Azure: {azure_count}" ) + if csp_breakdown: + print(" by region and service:") + for (provider, region, service), count in sorted( + csp_breakdown.items(), key=lambda kv: (-kv[1], kv[0]) + ): + print(f" {count:>4} {provider} {region} {service}") print(thin) if total_candidates == 0: diff --git a/imports/cloud_ip_ranges.py b/imports/cloud_ip_ranges.py index bf0b110..4b93fd2 100644 --- a/imports/cloud_ip_ranges.py +++ b/imports/cloud_ip_ranges.py @@ -4,7 +4,7 @@ import json import os import re -from typing import List, Tuple +from typing import Dict, List, Tuple from urllib.request import urlopen import requests @@ -13,82 +13,96 @@ IPV6_KEYWORDS = ["ipv6Prefix", "ipv6_prefix", "addressPrefixes"] -def fetch_ip_ranges_for_azure(url: str, extreme: bool) -> Tuple[List, List]: +def fetch_ip_ranges_for_azure(url: str, extreme: bool) -> Tuple[List, List, Dict]: try: response = requests.get(url, timeout=10) if response.status_code != 200: print( f"Failed to fetch IP ranges for Azure. Status code: {response.status_code}" ) - return [], [] + return [], [], {} data = json.loads(response.text) - ipv4_ranges = [ - item - for value in data.get("values", []) - for item in value.get("properties", {}).get("addressPrefixes", []) - if ":" not in item # Exclude IPv6 addresses - ] - ipv6_ranges = [ - item - for value in data.get("values", []) - for item in value.get("properties", {}).get("addressPrefixes", []) - if ":" in item # Only include IPv6 addresses - ] + ipv4_ranges = [] + ipv6_ranges = [] + metadata = {} + + for value in data.get("values", []): + props = value.get("properties", {}) + # Global tags carry an empty region; say so rather than inventing one. + region = props.get("region") or "global" + service = props.get("systemService") or value.get("name") or "unknown" + for item in props.get("addressPrefixes", []): + if ":" in item: + ipv6_ranges.append(item) + else: + ipv4_ranges.append(item) + metadata[item] = (region, service) + if extreme: print("IPv4 Ranges:", ipv4_ranges) print("IPv6 Ranges:", ipv6_ranges) - return ipv4_ranges, ipv6_ranges + return ipv4_ranges, ipv6_ranges, metadata except requests.exceptions.RequestException as e: print(f"An error occurred while fetching the IP ranges: {e}") - return [], [] + return [], [], {} -def fetch_ip_ranges(url: str, extreme: bool = False) -> Tuple[List, List]: +def fetch_ip_ranges(url: str, extreme: bool = False) -> Tuple[List, List, Dict]: try: response = requests.get(url, timeout=10) if response.status_code != 200: print(f"Failed to fetch IP ranges. Status code: {response.status_code}") - return [], [] + return [], [], {} data = json.loads(response.text) if "prefixes" not in data: print(f"No 'prefixes' key in retrieved data: {data}") - return [], [] - - ipv4_ranges = [ - prefix[keyword] - for prefix in data["prefixes"] - for keyword in IPV4_KEYWORDS - if keyword in prefix - ] - ipv6_ranges = [ - prefix[keyword] - for prefix in data["prefixes"] - for keyword in IPV6_KEYWORDS - if keyword in prefix - ] + return [], [], {} + + ipv4_ranges = [] + ipv6_ranges = [] + metadata = {} + + for prefix in data["prefixes"]: + # AWS calls it 'region', GCP calls it 'scope'. Both name the place an + # address is allocated from, which is what an operator needs in order + # to act on a match. + region = prefix.get("region") or prefix.get("scope") or "unknown" + service = prefix.get("service") or "unknown" + + for keyword in IPV4_KEYWORDS: + if keyword in prefix: + cidr = prefix[keyword] + ipv4_ranges.append(cidr) + metadata[cidr] = (region, service) + for keyword in IPV6_KEYWORDS: + if keyword in prefix: + cidr = prefix[keyword] + ipv6_ranges.append(cidr) + metadata[cidr] = (region, service) + if extreme: print("IPv4 Ranges:", ipv4_ranges) print("IPv6 Ranges:", ipv6_ranges) - return ipv4_ranges, ipv6_ranges + return ipv4_ranges, ipv6_ranges, metadata except requests.exceptions.RequestException as e: print(f"An error occurred while fetching the IP ranges: {e}") except IOError as e: print(f"An error occurred while writing to the file: {e}") - return [], [] + return [], [], {} def _fetch_and_save( url: str, filename: str, output_dir: str, extreme: bool -) -> Tuple[List, List]: +) -> Tuple[List, List, Dict]: ranges = fetch_ip_ranges(url, extreme) with open(os.path.join(output_dir, filename), "w", encoding="utf-8") as f: json.dump(ranges, f, indent=4) @@ -97,7 +111,7 @@ def _fetch_and_save( def fetch_google_cloud_ip_ranges( output_dir: str, extreme: bool = False -) -> Tuple[List, List]: +) -> Tuple[List, List, Dict]: return _fetch_and_save( "https://www.gstatic.com/ipranges/cloud.json", "gcp_ip_ranges.json", @@ -106,7 +120,9 @@ def fetch_google_cloud_ip_ranges( ) -def fetch_aws_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, List]: +def fetch_aws_ip_ranges( + output_dir: str, extreme: bool = False +) -> Tuple[List, List, Dict]: return _fetch_and_save( "https://ip-ranges.amazonaws.com/ip-ranges.json", "aws_ip_ranges.json", @@ -123,7 +139,7 @@ def fetch_aws_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, L AZURE_CACHE_PATH = ".azure_ip_cache.json" -def _save_azure_cache(ranges: Tuple[List, List]) -> None: +def _save_azure_cache(ranges: Tuple[List, List, Dict]) -> None: try: with open(AZURE_CACHE_PATH, "w", encoding="utf-8") as f: json.dump(ranges, f, indent=4) @@ -131,13 +147,18 @@ def _save_azure_cache(ranges: Tuple[List, List]) -> None: print(f"Warning: could not write Azure IP cache: {e}") -def _load_azure_cache() -> Tuple[List, List]: +def _load_azure_cache() -> Tuple[List, List, Dict]: with open(AZURE_CACHE_PATH, "r", encoding="utf-8") as f: data = json.load(f) - return data[0], data[1] + # Caches written before region/service capture hold only the two range lists. + # Read them rather than discarding a usable cache; metadata is simply absent. + metadata = data[2] if len(data) > 2 else {} + return data[0], data[1], metadata -def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, List]: +def fetch_azure_ip_ranges( + output_dir: str, extreme: bool = False +) -> Tuple[List, List, Dict]: confirmation_url = ( "https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519" ) @@ -160,7 +181,7 @@ def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, # Step 2: if scrape succeeded, fetch and return if json_url: ranges = fetch_ip_ranges_for_azure(json_url, extreme) - if ranges != ([], []): + if ranges[0] or ranges[1]: _save_azure_cache(ranges) with open( os.path.join(output_dir, "azure_ip_ranges.json"), "w", encoding="utf-8" @@ -185,7 +206,7 @@ def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, "This may be stale — update AZURE_PINNED_URL in cloud_ip_ranges.py if needed." ) ranges = fetch_ip_ranges_for_azure(AZURE_PINNED_URL, extreme) - if ranges != ([], []): + if ranges[0] or ranges[1]: _save_azure_cache(ranges) with open( os.path.join(output_dir, "azure_ip_ranges.json"), "w", encoding="utf-8" @@ -194,4 +215,4 @@ def fetch_azure_ip_ranges(output_dir: str, extreme: bool = False) -> Tuple[List, return ranges print("Azure IP fetch: all sources exhausted — no Azure ranges loaded.") - return [], [] + return [], [], {} diff --git a/imports/cloud_service_provider_checks.py b/imports/cloud_service_provider_checks.py index 763e6b4..a671ab9 100644 --- a/imports/cloud_service_provider_checks.py +++ b/imports/cloud_service_provider_checks.py @@ -75,7 +75,7 @@ def get_vendor_ips(domain_context, ip_version): def get_ip_matches(final_ips, vendor_ips_context, domain_context, ip_version): - matches = {vendor: set() for vendor in vendor_ips_context} + matches = {vendor: {} for vendor in vendor_ips_context} for ip in final_ips: if not is_ip_version(ip, ip_version): continue @@ -102,12 +102,14 @@ def match_ip_with_vendors(ip_obj, vendor_ips_context, domain_context, matches): domain_context.log_info( f"IP {ip_obj} is in range {ip_range} for vendor {vendor}" ) - matches[vendor].add(str(ip_obj)) + # Keep the prefix that matched — it is the key to the region and + # service the provider published for it. + matches[vendor][str(ip_obj)] = ip_range def merge_matches(matches_ipv4, matches_ipv6, vendor_ips_context): return { - vendor: list(matches_ipv4[vendor] | matches_ipv6[vendor]) + vendor: {**matches_ipv4[vendor], **matches_ipv6[vendor]} for vendor in vendor_ips_context } @@ -115,18 +117,33 @@ def merge_matches(matches_ipv4, matches_ipv6, vendor_ips_context): def log_and_write( vendor, matched_ips, domain, output_files, domain_context, written_lines ): - message = f"{domain} resolved to {vendor} IPs: {matched_ips}" + """ + Write one line per matched address, as a handoff record for downstream + tooling: domain|ip|provider|region|service|prefix + + One address per line, pipe-delimited, because this file is consumed by + another tool rather than read as prose. Region and service come from the + provider's own published ranges and are what make a match actionable — an + address is only worth pursuing if you know where it is allocated from and + what it belongs to. + """ + csp_ip_addresses = domain_context.get_csp_ip_addresses() + file_path = output_files["standard"]["csp"] + wrote_any = False - # Deduplicate against an in-memory, run-scoped set of lines already - # written — avoids re-reading the whole output file on every call. - if message in written_lines: - return False + for ip, prefix in sorted(matched_ips.items()): + region, service = csp_ip_addresses.describe(prefix) + message = f"{domain}|{ip}|{vendor}|{region}|{service}|{prefix}" - file_path = output_files["standard"]["csp"] - with open(file_path, "a", encoding="utf-8") as file: - file.write(message + "\n") - written_lines.add(message) + # Deduplicate against an in-memory, run-scoped set of lines already + # written — avoids re-reading the whole output file on every call. + if message in written_lines: + continue - domain_context.log_info(message) + with open(file_path, "a", encoding="utf-8") as file: + file.write(message + "\n") + written_lines.add(message) + domain_context.log_info(message) + wrote_any = True - return True + return wrote_any diff --git a/resolver.py b/resolver.py index c05483b..f66b85c 100644 --- a/resolver.py +++ b/resolver.py @@ -21,18 +21,24 @@ async def run(env_manager): both the CLI entrypoint (resolver.py) and the Lambda entrypoint (lambda_handler.py) can share the same logic. """ - gcp_ipv4, gcp_ipv6 = fetch_google_cloud_ip_ranges( + gcp_ipv4, gcp_ipv6, gcp_meta = fetch_google_cloud_ip_ranges( env_manager.output_dir, env_manager.extreme ) - aws_ipv4, aws_ipv6 = fetch_aws_ip_ranges( + aws_ipv4, aws_ipv6, aws_meta = fetch_aws_ip_ranges( env_manager.output_dir, env_manager.extreme ) - azure_ipv4, azure_ipv6 = fetch_azure_ip_ranges( + azure_ipv4, azure_ipv6, azure_meta = fetch_azure_ip_ranges( env_manager.output_dir, env_manager.extreme ) csp_ip_addresses = CSPIPAddresses( - gcp_ipv4, gcp_ipv6, aws_ipv4, aws_ipv6, azure_ipv4, azure_ipv6 + gcp_ipv4, + gcp_ipv6, + aws_ipv4, + aws_ipv6, + azure_ipv4, + azure_ipv6, + metadata={**gcp_meta, **aws_meta, **azure_meta}, ) env_manager.set_domains() diff --git a/tests/conftest.py b/tests/conftest.py index 77bb8c6..871cb97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,12 +23,33 @@ AZURE_IPV4 = ["13.0.0.0/8", "20.0.0.0/8"] AZURE_IPV6 = ["2603:1000::/24"] +# CIDR -> (region, service), as the providers publish it. A match is only +# actionable downstream if the consumer knows where the address is allocated +# from and what it belongs to. +CSP_METADATA = { + "34.0.0.0/8": ("europe-west2", "Google Cloud"), + "35.0.0.0/8": ("us-central1", "Google Cloud"), + "2600:1900::/35": ("europe-west2", "Google Cloud"), + "3.0.0.0/8": ("eu-west-2", "EC2"), + "52.0.0.0/8": ("us-east-1", "AMAZON"), + "2600:1f00::/25": ("eu-west-2", "EC2"), + "13.0.0.0/8": ("uksouth", "AzureCloud"), + "20.0.0.0/8": ("global", "AzureCloud"), + "2603:1000::/24": ("uksouth", "AzureCloud"), +} + @pytest.fixture def csp_ips(): """A real CSPIPAddresses instance populated with test ranges.""" return CSPIPAddresses( - GCP_IPV4, GCP_IPV6, AWS_IPV4, AWS_IPV6, AZURE_IPV4, AZURE_IPV6 + GCP_IPV4, + GCP_IPV6, + AWS_IPV4, + AWS_IPV6, + AZURE_IPV4, + AZURE_IPV6, + metadata=CSP_METADATA, ) diff --git a/tests/test_cloud_ip_ranges.py b/tests/test_cloud_ip_ranges.py index 4f2ef93..1f23fb7 100644 --- a/tests/test_cloud_ip_ranges.py +++ b/tests/test_cloud_ip_ranges.py @@ -43,7 +43,7 @@ def test_fetch_ip_ranges_returns_ipv4_and_ipv6(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=GCP_PAYLOAD), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert "34.0.0.0/8" in v4 assert "2600:1900::/28" in v6 @@ -54,7 +54,7 @@ def test_fetch_ip_ranges_non_200_returns_empty(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(status_code=503), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert v4 == [] assert v6 == [] @@ -65,7 +65,7 @@ def test_fetch_ip_ranges_missing_prefixes_key_returns_empty(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body={"other": []}), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert v4 == [] assert v6 == [] @@ -76,7 +76,7 @@ def test_fetch_ip_ranges_request_exception_returns_empty(): "imports.cloud_ip_ranges.requests.get", side_effect=requests.exceptions.RequestException("timeout"), ): - v4, v6 = fetch_ip_ranges("http://fake-url") + v4, v6, _meta = fetch_ip_ranges("http://fake-url") assert v4 == [] assert v6 == [] @@ -87,7 +87,7 @@ def test_fetch_ip_ranges_extreme_flag_does_not_raise(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=GCP_PAYLOAD), ): - v4, v6 = fetch_ip_ranges("http://fake-url", extreme=True) + v4, v6, _meta = fetch_ip_ranges("http://fake-url", extreme=True) assert "34.0.0.0/8" in v4 @@ -106,7 +106,9 @@ def test_fetch_ip_ranges_for_azure_splits_ipv4_and_ipv6(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=AZURE_PAYLOAD), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=False) + v4, v6, _meta = fetch_ip_ranges_for_azure( + "http://fake-azure-url", extreme=False + ) assert "13.64.0.0/11" in v4 assert "2603:1010::/48" in v6 @@ -117,7 +119,9 @@ def test_fetch_ip_ranges_for_azure_non_200_returns_empty(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(status_code=404), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=False) + v4, v6, _meta = fetch_ip_ranges_for_azure( + "http://fake-azure-url", extreme=False + ) assert v4 == [] assert v6 == [] @@ -128,7 +132,9 @@ def test_fetch_ip_ranges_for_azure_request_exception_returns_empty(): "imports.cloud_ip_ranges.requests.get", side_effect=requests.exceptions.RequestException("timeout"), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=False) + v4, v6, _meta = fetch_ip_ranges_for_azure( + "http://fake-azure-url", extreme=False + ) assert v4 == [] assert v6 == [] @@ -139,7 +145,7 @@ def test_fetch_ip_ranges_for_azure_extreme_does_not_raise(): "imports.cloud_ip_ranges.requests.get", return_value=_mock_response(body=AZURE_PAYLOAD), ): - v4, v6 = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=True) + v4, v6, _meta = fetch_ip_ranges_for_azure("http://fake-azure-url", extreme=True) assert "13.64.0.0/11" in v4 @@ -246,4 +252,122 @@ def test_fetch_azure_ip_ranges_all_sources_fail_returns_empty(tmp_path): ): result = fetch_azure_ip_ranges(str(tmp_path)) - assert result == ([], []) + assert result[:2] == ([], []) + + +# --------------------------------------------------------------------------- +# Region / service metadata capture +# +# A cloud match is only actionable downstream if the consumer knows where the +# address is allocated from and what it belongs to. All three providers publish +# that alongside the prefix, so it must survive parsing. +# --------------------------------------------------------------------------- + + +def test_aws_region_and_service_are_captured(monkeypatch): + """AWS publishes region and service per prefix; both must be retained.""" + payload = { + "prefixes": [ + {"ip_prefix": "3.5.140.0/22", "region": "eu-west-2", "service": "EC2"}, + {"ip_prefix": "52.94.0.0/22", "region": "us-east-1", "service": "AMAZON"}, + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/aws.json") + + assert "3.5.140.0/22" in ipv4 + assert meta["3.5.140.0/22"] == ("eu-west-2", "EC2") + assert meta["52.94.0.0/22"] == ("us-east-1", "AMAZON") + + +def test_gcp_scope_is_captured_as_region(monkeypatch): + """GCP calls it 'scope'; it means the same thing and must map through.""" + payload = { + "prefixes": [ + { + "ipv4Prefix": "34.1.208.0/20", + "service": "Google Cloud", + "scope": "europe-west2", + } + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + _ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/gcp.json") + + assert meta["34.1.208.0/20"] == ("europe-west2", "Google Cloud") + + +def test_missing_metadata_is_reported_as_unknown_not_invented(monkeypatch): + """A prefix without region/service must say so rather than guess.""" + payload = {"prefixes": [{"ip_prefix": "198.51.100.0/24"}]} + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + _ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/x.json") + + assert meta["198.51.100.0/24"] == ("unknown", "unknown") + + +def test_azure_region_and_system_service_are_captured(monkeypatch): + """Azure carries region and systemService on the tag's properties.""" + payload = { + "values": [ + { + "name": "AzureCloud.uksouth", + "properties": { + "region": "uksouth", + "systemService": "AzureCloud", + "addressPrefixes": ["20.26.0.0/16", "2603:1000::/24"], + }, + } + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + ipv4, ipv6, meta = fetch_ip_ranges_for_azure( + "http://example.invalid/az.json", False + ) + + assert ipv4 == ["20.26.0.0/16"] + assert ipv6 == ["2603:1000::/24"] + assert meta["20.26.0.0/16"] == ("uksouth", "AzureCloud") + assert meta["2603:1000::/24"] == ("uksouth", "AzureCloud") + + +def test_azure_global_tag_region_is_labelled_global(monkeypatch): + """Azure leaves region empty on global tags; that must not read as unknown.""" + payload = { + "values": [ + { + "name": "AzureCloud", + "properties": { + "region": "", + "systemService": "", + "addressPrefixes": ["13.64.0.0/16"], + }, + } + ] + } + monkeypatch.setattr( + "imports.cloud_ip_ranges.requests.get", + lambda *a, **k: _mock_response(200, payload), + ) + + _ipv4, _ipv6, meta = fetch_ip_ranges_for_azure( + "http://example.invalid/az.json", False + ) + + assert meta["13.64.0.0/16"] == ("global", "AzureCloud") diff --git a/tests/test_cloud_service_provider_checks.py b/tests/test_cloud_service_provider_checks.py index 825b39e..92e412a 100644 --- a/tests/test_cloud_service_provider_checks.py +++ b/tests/test_cloud_service_provider_checks.py @@ -86,33 +86,35 @@ def test_get_vendor_ips_unknown_version_returns_empty(ctx): def test_match_ip_with_vendors_records_match(ctx): ip_obj = ipaddress.IPv4Address("34.1.2.3") vendor_ips = {"gcp": ["34.0.0.0/8"], "aws": [], "azure": []} - matches = {"gcp": set(), "aws": set(), "azure": set()} + matches = {"gcp": {}, "aws": {}, "azure": {}} match_ip_with_vendors(ip_obj, vendor_ips, ctx, matches) assert "34.1.2.3" in matches["gcp"] - assert matches["aws"] == set() + # The matched prefix is retained — it is the key to region and service. + assert matches["gcp"]["34.1.2.3"] == "34.0.0.0/8" + assert matches["aws"] == {} def test_match_ip_with_vendors_no_match(ctx): ip_obj = ipaddress.IPv4Address("1.2.3.4") vendor_ips = {"gcp": ["34.0.0.0/8"]} - matches = {"gcp": set()} + matches = {"gcp": {}} match_ip_with_vendors(ip_obj, vendor_ips, ctx, matches) - assert matches["gcp"] == set() + assert matches["gcp"] == {} def test_match_ip_with_vendors_ignores_invalid_range(ctx): """A bad CIDR string should not crash — it's logged and skipped.""" ip_obj = ipaddress.IPv4Address("1.2.3.4") vendor_ips = {"gcp": ["not-a-valid-cidr"]} - matches = {"gcp": set()} + matches = {"gcp": {}} match_ip_with_vendors(ip_obj, vendor_ips, ctx, matches) - assert matches["gcp"] == set() + assert matches["gcp"] == {} # --------------------------------------------------------------------------- @@ -121,21 +123,22 @@ def test_match_ip_with_vendors_ignores_invalid_range(ctx): def test_merge_matches_combines_ipv4_and_ipv6(): - v4 = {"gcp": {"1.2.3.4"}, "aws": set()} - v6 = {"gcp": {"::1"}, "aws": set()} + """Both families merge, each address keeping the prefix it matched.""" + v4 = {"gcp": {"1.2.3.4": "1.2.0.0/16"}, "aws": {}} + v6 = {"gcp": {"::1": "::/64"}, "aws": {}} vendor_context = {"gcp": [], "aws": []} result = merge_matches(v4, v6, vendor_context) - assert set(result["gcp"]) == {"1.2.3.4", "::1"} - assert result["aws"] == [] + assert result["gcp"] == {"1.2.3.4": "1.2.0.0/16", "::1": "::/64"} + assert result["aws"] == {} def test_merge_matches_empty_sets(): - v4 = {"gcp": set()} - v6 = {"gcp": set()} + v4 = {"gcp": {}} + v6 = {"gcp": {}} result = merge_matches(v4, v6, {"gcp": []}) - assert result["gcp"] == [] + assert result["gcp"] == {} # --------------------------------------------------------------------------- @@ -146,13 +149,13 @@ def test_merge_matches_empty_sets(): def test_get_ip_matches_finds_gcp_ipv4(ctx, csp_ips): # 34.1.2.3 falls in GCP's 34.0.0.0/8 range (from conftest) result = get_ip_matches(["34.1.2.3"], get_vendor_ips(ctx, 4), ctx, ip_version=4) - assert "34.1.2.3" in result["gcp"] + assert result["gcp"]["34.1.2.3"] == "34.0.0.0/8" def test_get_ip_matches_skips_wrong_ip_version(ctx): # An IPv6 address should be skipped when looking for IPv4 matches result = get_ip_matches(["2600:1900::1"], get_vendor_ips(ctx, 4), ctx, ip_version=4) - assert all(len(s) == 0 for s in result.values()) + assert all(len(m) == 0 for m in result.values()) # --------------------------------------------------------------------------- @@ -165,10 +168,17 @@ def test_log_and_write_creates_entry(tmp_path, ctx): out_file.touch() output_files = {"standard": {"csp": str(out_file)}} - result = log_and_write("gcp", ["34.1.2.3"], "example.com", output_files, ctx, set()) + result = log_and_write( + "gcp", {"34.1.2.3": "34.0.0.0/8"}, "example.com", output_files, ctx, set() + ) assert result is True - assert "example.com resolved to gcp IPs" in out_file.read_text() + # One handoff record per address: domain|ip|provider|region|service|prefix. + # Region and service are what make the match actionable downstream. + assert ( + out_file.read_text().strip() + == "example.com|34.1.2.3|gcp|europe-west2|Google Cloud|34.0.0.0/8" + ) def test_log_and_write_no_duplicate_entries(tmp_path, ctx): @@ -179,10 +189,20 @@ def test_log_and_write_no_duplicate_entries(tmp_path, ctx): written_lines = set() first = log_and_write( - "gcp", ["34.1.2.3"], "example.com", output_files, ctx, written_lines + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, ) second = log_and_write( - "gcp", ["34.1.2.3"], "example.com", output_files, ctx, written_lines + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, ) assert first is True @@ -207,9 +227,16 @@ def guarded_open(file, mode="r", *args, **kwargs): monkeypatch.setattr("builtins.open", guarded_open) - log_and_write("gcp", ["34.1.2.3"], "example.com", output_files, ctx, written_lines) + log_and_write( + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, + ) - assert "example.com resolved to gcp IPs" in out_file.read_text() + assert "example.com|34.1.2.3|gcp|" in out_file.read_text() def test_log_and_write_substring_line_not_suppressed(tmp_path, ctx): @@ -221,14 +248,18 @@ def test_log_and_write_substring_line_not_suppressed(tmp_path, ctx): output_files = {"standard": {"csp": str(out_file)}} written_lines = set() - # "example.com resolved to gcp IPs: ['1.1.1.1']" contains - # "ple.com resolved to gcp IPs: ['1.1.1.1']" as a substring + # "example.com|34.1.2.3|..." contains "ple.com|34.1.2.3|..." as a substring # (example.com = exam + ple.com), but they are different domains/lines. first = log_and_write( - "gcp", ["1.1.1.1"], "example.com", output_files, ctx, written_lines + "gcp", + {"34.1.2.3": "34.0.0.0/8"}, + "example.com", + output_files, + ctx, + written_lines, ) second = log_and_write( - "gcp", ["1.1.1.1"], "ple.com", output_files, ctx, written_lines + "gcp", {"34.1.2.3": "34.0.0.0/8"}, "ple.com", output_files, ctx, written_lines ) assert first is True @@ -316,11 +347,11 @@ def test_parse_network_equivalence_ip_inside_and_outside_range(ctx): inside_ip = ipaddress.IPv4Address("34.1.2.3") outside_ip = ipaddress.IPv4Address("8.8.8.8") - matches = {"gcp": set(), "aws": set(), "azure": set()} + matches = {"gcp": {}, "aws": {}, "azure": {}} match_ip_with_vendors(inside_ip, vendor_ips, ctx, matches) match_ip_with_vendors(outside_ip, vendor_ips, ctx, matches) - assert matches == {"gcp": {"34.1.2.3"}, "aws": set(), "azure": set()} + assert matches == {"gcp": {"34.1.2.3": "34.0.0.0/8"}, "aws": {}, "azure": {}} def test_parse_network_is_memoised_across_calls(): @@ -348,7 +379,7 @@ def test_match_ip_with_vendors_reuses_cached_network(ctx): re-parse the CIDR strings for each IP.""" parse_network.cache_clear() vendor_ips = {"gcp": ["34.0.0.0/8", "35.0.0.0/8"], "aws": [], "azure": []} - matches = {"gcp": set(), "aws": set(), "azure": set()} + matches = {"gcp": {}, "aws": {}, "azure": {}} ips = [ipaddress.IPv4Address(f"34.1.2.{i}") for i in range(10)] for ip in ips: @@ -359,3 +390,44 @@ def test_match_ip_with_vendors_reuses_cached_network(ctx): # regardless of how many IPs (10) were checked against them. assert info.misses == 2 assert info.hits == 10 * 2 - 2 + + +def test_handoff_record_carries_region_and_service_end_to_end( + tmp_path, mock_env_manager, ctx +): + """ + The whole point of the CSP output: a downstream tool must be able to read + where an address is allocated from and what it belongs to, without going + back to the provider's range files. + """ + out_file = tmp_path / "csp.txt" + out_file.touch() + mock_env_manager.output_files = {"standard": {"csp": str(out_file)}} + + # 3.1.2.3 falls in AWS 3.0.0.0/8, published as eu-west-2 / EC2 in conftest. + perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3"]) + + line = out_file.read_text().strip() + assert line == "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8" + + domain, ip, provider, region, service, prefix = line.split("|") + assert (provider, region, service) == ("aws", "eu-west-2", "EC2") + + +def test_each_matched_address_gets_its_own_record(tmp_path, mock_env_manager, ctx): + """ + One line per address, not one line per domain with a list. Addresses in the + same provider can sit in different regions, and a consumer acts per address. + """ + out_file = tmp_path / "csp.txt" + out_file.touch() + mock_env_manager.output_files = {"standard": {"csp": str(out_file)}} + + # 3.x -> eu-west-2/EC2, 52.x -> us-east-1/AMAZON (both AWS, different regions) + perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3", "52.1.2.3"]) + + lines = sorted(ln for ln in out_file.read_text().splitlines() if ln) + assert lines == [ + "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8", + "example.com|52.1.2.3|aws|us-east-1|AMAZON|52.0.0.0/8", + ] diff --git a/tests/test_run_summary.py b/tests/test_run_summary.py index 2b0fee3..149d465 100644 --- a/tests/test_run_summary.py +++ b/tests/test_run_summary.py @@ -45,9 +45,9 @@ def test_counts_are_accurate(tmp_path, capsys): "resolved": "a.com|1.1.1.1\nb.com|2.2.2.2\n", "unresolved": "DNS resolution error for c.com: timeout\n", "csp": ( - "a.com resolved to aws IPs: ['1.1.1.1']\n" - "b.com resolved to gcp IPs: ['2.2.2.2']\n" - "b2.com resolved to gcp IPs: ['3.3.3.3']\n" + "a.com|1.1.1.1|aws|eu-west-2|EC2|1.1.0.0/16\n" + "b.com|2.2.2.2|gcp|europe-west2|Google Cloud|2.2.0.0/16\n" + "b2.com|3.3.3.3|gcp|us-central1|Google Cloud|3.3.0.0/16\n" ), }, ) @@ -212,3 +212,28 @@ def test_missing_file_treated_as_empty(tmp_path, capsys): out = capsys.readouterr().out assert "No takeover candidates detected." in out + + +def test_csp_breakdown_shows_region_and_service(tmp_path, capsys): + """ + A provider tally alone is not actionable — an operator needs the region an + address is allocated from and the service it belongs to. + """ + summary = _make_summary( + tmp_path, + { + "csp": ( + "a.com|1.1.1.1|aws|eu-west-2|EC2|1.1.0.0/16\n" + "b.com|1.1.1.2|aws|eu-west-2|EC2|1.1.0.0/16\n" + "c.com|9.9.9.9|aws|us-east-1|AMAZON|9.9.0.0/16\n" + ), + }, + ) + summary.display(total_input=3, failed_count=0) + + out = capsys.readouterr().out + assert "by region and service" in out + assert "aws eu-west-2 EC2" in out + assert "aws us-east-1 AMAZON" in out + # The busiest grouping is listed first. + assert out.index("eu-west-2") < out.index("us-east-1")