diff --git a/AGENTS.md b/AGENTS.md index 11c4fee..1a7e7fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,6 +171,17 @@ bash ~/.claude/skills/local-ci/local-ci.sh --workflow ci.yml name under the same platform return byte-identical DNS. Distinguishing them needs an HTTP request, which §1 rules out. The flag means *"resolution proves nothing in this zone"*, not *"this host does not exist"*. +- **Wildcard detection is best-effort against rotating pools.** The check asks whether a + resolution falls entirely inside the addresses a random-label probe returned. Where the + catch-all is a large load-balanced fleet, two probes see only part of the rotation and a + later name lands outside the observed set, so it is not recognised. Verified live against + a platform whose wildcard rotates across regions: of two fabricated names, one was marked + and one was not. A match is good evidence; a miss is not evidence of absence. + + A zone-level semantic (mark every resolution in a zone known to answer for anything, rather + than testing each address against a sampled set) would be more robust and is arguably more + honest, since DNS cannot discriminate within such a zone anyway. That is an open design + decision, not an oversight. - **Performance at scale is reasoned, not measured.** Known cliffs were removed (per-IP CIDR parsing, O(n²) dedupe) but no benchmark has been run against a large list. diff --git a/README.md b/README.md index 0d7efdf..af00828 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Each run creates a timestamped subdirectory under the output directory containin | `resolution_results_*.txt` | Successfully resolved domains and their IPv4/IPv6 addresses, pipe-delimited (`domain\|ip1\|ip2`). Lines prefixed `WILDCARD\|` resolved only via a zone wildcard — see [Wildcard DNS detection](#wildcard-dns-detection) | | `unresolved_results_*.txt` | Domains that could not be resolved after all retries | | `takeover_candidates_*.txt` | Takeover candidates — `DANGLING\|` lines (dangling CNAMEs with category, recommendation, evidence) and `NS_TAKEOVER\|` lines (unresolvable nameservers) | -| `csp_matches_*.txt` | One handoff record per matched address: `domain\|ip\|provider\|region\|service\|prefix`. Region and service come from the provider's own published ranges — see [Cloud IP attribution](#cloud-ip-attribution) | +| `csp_matches_*.txt` | One handoff record per matched address: `domain\|ip\|provider\|region\|service\|prefix\|border_group`. Prefixed `WILDCARD\|` when the resolution was a catch-all. See [Cloud IP attribution](#cloud-ip-attribution) | | `environment_results_*.json` | Run metadata (command, external IP, Docker status) | | `evidence/dns/` | dig or nslookup output per flagged domain (when `--evidence` is set) | @@ -128,14 +128,23 @@ Each match is therefore written as a record carrying the provider's published re service: ``` -domain|ip|provider|region|service|prefix +domain|ip|provider|region|service|prefix|border_group ``` ``` -example.com|13.35.163.22|aws|GLOBAL|CLOUDFRONT|13.35.0.0/16 -example.com|3.11.53.7|aws|eu-west-2|EC2|3.8.0.0/14 +example.com|13.35.163.22|aws|GLOBAL|CLOUDFRONT|13.35.0.0/16|GLOBAL +example.com|3.11.53.7|aws|eu-west-2|EC2|3.8.0.0/14|eu-west-2 ``` +`border_group` is AWS's network border group: the boundary an Elastic IP is actually allocated +and advertised from. It usually mirrors the region, but differs for Local Zones and Wavelength — +which is precisely where the distinction matters. GCP and Azure publish no equivalent and it +reads `unknown` for them. + +Records for a resolution that came from a zone wildcard are prefixed `WILDCARD|`. Those addresses +belong to the hosting platform and are in active use, so they are not targets; they are reported +for completeness and excluded from the summary's counts. + The difference is the point: the first is a CDN edge address, the second an EC2 address in a specific region. Both are "AWS"; only one is a meaningful target. @@ -287,6 +296,11 @@ self-referential and non-existent CNAMEs, IPv6 (AAAA) resolution, and wildcard D - **Wildcard detection cannot separate a real host from a catch-all** when the host genuinely shares the wildcard's addresses (GitHub Pages is the common case). Confirming those requires active probing, which is out of scope. +- **Wildcard detection is unreliable against large rotating address pools.** Where a catch-all is + served by a big load-balanced fleet (Heroku, for example), two probes capture only part of the + rotation, so a later name resolving to different addresses in the same fleet is not recognised as + a wildcard answer. Detection is best-effort: a match is good evidence, a miss is not evidence of + absence. - **A takeover candidate is only recorded when a CNAME actually exists.** A name that simply does not resolve is reported as unresolved, not as a candidate — there is nothing to claim. diff --git a/classes/csp_ip_addresses.py b/classes/csp_ip_addresses.py index 2dc69da..5fca482 100644 --- a/classes/csp_ip_addresses.py +++ b/classes/csp_ip_addresses.py @@ -21,8 +21,13 @@ def __init__( self.metadata = metadata or {} def describe(self, cidr): - """Region and service for a matched prefix, or unknowns if unpublished.""" - return self.metadata.get(cidr, ("unknown", "unknown")) + """ + Region, service and network border group for a matched prefix. + + Unpublished fields read 'unknown' rather than being inferred — only AWS + publishes a border group, and not every prefix carries a region. + """ + return self.metadata.get(cidr, ("unknown", "unknown", "unknown")) def get_gcp_ipv4(self): return self.gcp_ipv4 diff --git a/classes/run_summary.py b/classes/run_summary.py index 52a41cd..3bc57e9 100644 --- a/classes/run_summary.py +++ b/classes/run_summary.py @@ -32,9 +32,20 @@ def display(self, total_input, failed_count): if ln.startswith("NS_TAKEOVER|") ] - # Handoff records: domain|ip|provider|region|service|prefix + # Handoff records: [WILDCARD|]domain|ip|provider|region|service|prefix|border csp_lines = self._lines("csp") - csp_records = [ln.split("|") for ln in csp_lines if ln.count("|") >= 5] + csp_records = [] + csp_wildcard_count = 0 + for line in csp_lines: + if line.startswith("WILDCARD|"): + csp_wildcard_count += 1 + # Counted, but kept out of the target breakdown below: these are + # the hosting platform's addresses, not claimable targets. + continue + fields = line.split("|") + if len(fields) >= 6: + csp_records.append(fields) + aws_count = sum(1 for r in csp_records if r[2] == "aws") gcp_count = sum(1 for r in csp_records if r[2] == "gcp") azure_count = sum(1 for r in csp_records if r[2] == "azure") @@ -85,6 +96,11 @@ def display(self, total_input, failed_count): print( f" CSP matches — AWS: {aws_count} GCP: {gcp_count} Azure: {azure_count}" ) + if csp_wildcard_count: + print( + f" excluded (wildcard) : {csp_wildcard_count:>4}" + " (hosting platform addresses, not targets)" + ) if csp_breakdown: print(" by region and service:") for (provider, region, service), count in sorted( diff --git a/imports/cloud_ip_ranges.py b/imports/cloud_ip_ranges.py index 4b93fd2..6c0a7ac 100644 --- a/imports/cloud_ip_ranges.py +++ b/imports/cloud_ip_ranges.py @@ -38,7 +38,7 @@ def fetch_ip_ranges_for_azure(url: str, extreme: bool) -> Tuple[List, List, Dict ipv6_ranges.append(item) else: ipv4_ranges.append(item) - metadata[item] = (region, service) + metadata[item] = (region, service, "unknown") if extreme: print("IPv4 Ranges:", ipv4_ranges) @@ -74,17 +74,22 @@ def fetch_ip_ranges(url: str, extreme: bool = False) -> Tuple[List, List, Dict]: # to act on a match. region = prefix.get("region") or prefix.get("scope") or "unknown" service = prefix.get("service") or "unknown" + # AWS publishes the network border group: the boundary an Elastic IP + # is actually allocated and advertised from. It usually mirrors the + # region but differs for Local Zones and Wavelength, which is exactly + # where the distinction matters. GCP and Azure publish no equivalent. + border_group = prefix.get("network_border_group") or "unknown" for keyword in IPV4_KEYWORDS: if keyword in prefix: cidr = prefix[keyword] ipv4_ranges.append(cidr) - metadata[cidr] = (region, service) + metadata[cidr] = (region, service, border_group) for keyword in IPV6_KEYWORDS: if keyword in prefix: cidr = prefix[keyword] ipv6_ranges.append(cidr) - metadata[cidr] = (region, service) + metadata[cidr] = (region, service, border_group) if extreme: print("IPv4 Ranges:", ipv4_ranges) diff --git a/imports/cloud_service_provider_checks.py b/imports/cloud_service_provider_checks.py index a671ab9..6bd144e 100644 --- a/imports/cloud_service_provider_checks.py +++ b/imports/cloud_service_provider_checks.py @@ -13,7 +13,7 @@ def parse_network(cidr): return ipaddress.ip_network(cidr) -def perform_csp_checks(domain_context, env_manager, final_ips): +def perform_csp_checks(domain_context, env_manager, final_ips, is_wildcard=False): domain = domain_context.get_domain() output_files = env_manager.output_files @@ -48,6 +48,7 @@ def perform_csp_checks(domain_context, env_manager, final_ips): output_files, domain_context, written_lines, + is_wildcard, ) or success ) @@ -115,25 +116,42 @@ def merge_matches(matches_ipv4, matches_ipv6, vendor_ips_context): def log_and_write( - vendor, matched_ips, domain, output_files, domain_context, written_lines + vendor, + matched_ips, + domain, + output_files, + domain_context, + written_lines, + is_wildcard=False, ): """ Write one line per matched address, as a handoff record for downstream - tooling: domain|ip|provider|region|service|prefix + tooling: + + domain|ip|provider|region|service|prefix|border_group One address per line, pipe-delimited, because this file is consumed by - another tool rather than read as prose. Region and service come from the - provider's own published ranges and are what make a match actionable — an - address is only worth pursuing if you know where it is allocated from and - what it belongs to. + another tool rather than read as prose. Region, service and border group + come from the provider's own published ranges and are what make a match + actionable — an address is only worth pursuing if you know where it is + allocated from and what it belongs to. + + Records for a wildcard resolution are prefixed `WILDCARD|`. Those addresses + belong to the hosting platform and are in active use, so they are the + opposite of a claimable target; without the marker a single wildcarded zone + emits one record per enumerated subdomain and buries the real findings. """ csp_ip_addresses = domain_context.get_csp_ip_addresses() file_path = output_files["standard"]["csp"] + line_prefix = "WILDCARD|" if is_wildcard else "" wrote_any = False for ip, prefix in sorted(matched_ips.items()): - region, service = csp_ip_addresses.describe(prefix) - message = f"{domain}|{ip}|{vendor}|{region}|{service}|{prefix}" + region, service, border_group = csp_ip_addresses.describe(prefix) + message = ( + f"{line_prefix}{domain}|{ip}|{vendor}|{region}|{service}" + f"|{prefix}|{border_group}" + ) # Deduplicate against an in-memory, run-scoped set of lines already # written — avoids re-reading the whole output file on every call. diff --git a/imports/domain_processor.py b/imports/domain_processor.py index 1961a2f..ff1af2f 100644 --- a/imports/domain_processor.py +++ b/imports/domain_processor.py @@ -29,7 +29,11 @@ async def process_domain_async( output_files["standard"]["resolved"], f"{prefix}{domain}|{'|'.join(final_ips)}", ) - perform_csp_checks(domain_context, env_manager, final_ips) + # The wildcard verdict travels with the cloud match too. A catch-all + # address belongs to the hosting platform and is in active use, so it is + # not a claimable target — but without the marker a single wildcarded + # zone emits one cloud record per enumerated subdomain. + perform_csp_checks(domain_context, env_manager, final_ips, is_wildcard) env_manager.log_info(f"Performing CSP Checks for: {domain} and {final_ips}") pbar.update(1) diff --git a/tests/conftest.py b/tests/conftest.py index 871cb97..7950010 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,16 +26,18 @@ # CIDR -> (region, service), as the providers publish it. A match is only # actionable downstream if the consumer knows where the address is allocated # from and what it belongs to. +# CIDR -> (region, service, network_border_group). Only AWS publishes a border +# group; it is the boundary an Elastic IP is actually allocated from. CSP_METADATA = { - "34.0.0.0/8": ("europe-west2", "Google Cloud"), - "35.0.0.0/8": ("us-central1", "Google Cloud"), - "2600:1900::/35": ("europe-west2", "Google Cloud"), - "3.0.0.0/8": ("eu-west-2", "EC2"), - "52.0.0.0/8": ("us-east-1", "AMAZON"), - "2600:1f00::/25": ("eu-west-2", "EC2"), - "13.0.0.0/8": ("uksouth", "AzureCloud"), - "20.0.0.0/8": ("global", "AzureCloud"), - "2603:1000::/24": ("uksouth", "AzureCloud"), + "34.0.0.0/8": ("europe-west2", "Google Cloud", "unknown"), + "35.0.0.0/8": ("us-central1", "Google Cloud", "unknown"), + "2600:1900::/35": ("europe-west2", "Google Cloud", "unknown"), + "3.0.0.0/8": ("eu-west-2", "EC2", "eu-west-2"), + "52.0.0.0/8": ("us-east-1", "AMAZON", "us-east-1"), + "2600:1f00::/25": ("eu-west-2", "EC2", "eu-west-2"), + "13.0.0.0/8": ("uksouth", "AzureCloud", "unknown"), + "20.0.0.0/8": ("global", "AzureCloud", "unknown"), + "2603:1000::/24": ("uksouth", "AzureCloud", "unknown"), } diff --git a/tests/test_cloud_ip_ranges.py b/tests/test_cloud_ip_ranges.py index 1f23fb7..bd38894 100644 --- a/tests/test_cloud_ip_ranges.py +++ b/tests/test_cloud_ip_ranges.py @@ -268,7 +268,12 @@ def test_aws_region_and_service_are_captured(monkeypatch): """AWS publishes region and service per prefix; both must be retained.""" payload = { "prefixes": [ - {"ip_prefix": "3.5.140.0/22", "region": "eu-west-2", "service": "EC2"}, + { + "ip_prefix": "3.5.140.0/22", + "region": "eu-west-2", + "service": "EC2", + "network_border_group": "eu-west-2", + }, {"ip_prefix": "52.94.0.0/22", "region": "us-east-1", "service": "AMAZON"}, ] } @@ -280,8 +285,8 @@ def test_aws_region_and_service_are_captured(monkeypatch): ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/aws.json") assert "3.5.140.0/22" in ipv4 - assert meta["3.5.140.0/22"] == ("eu-west-2", "EC2") - assert meta["52.94.0.0/22"] == ("us-east-1", "AMAZON") + assert meta["3.5.140.0/22"] == ("eu-west-2", "EC2", "eu-west-2") + assert meta["52.94.0.0/22"] == ("us-east-1", "AMAZON", "unknown") def test_gcp_scope_is_captured_as_region(monkeypatch): @@ -302,7 +307,7 @@ def test_gcp_scope_is_captured_as_region(monkeypatch): _ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/gcp.json") - assert meta["34.1.208.0/20"] == ("europe-west2", "Google Cloud") + assert meta["34.1.208.0/20"] == ("europe-west2", "Google Cloud", "unknown") def test_missing_metadata_is_reported_as_unknown_not_invented(monkeypatch): @@ -315,7 +320,7 @@ def test_missing_metadata_is_reported_as_unknown_not_invented(monkeypatch): _ipv4, _ipv6, meta = fetch_ip_ranges("http://example.invalid/x.json") - assert meta["198.51.100.0/24"] == ("unknown", "unknown") + assert meta["198.51.100.0/24"] == ("unknown", "unknown", "unknown") def test_azure_region_and_system_service_are_captured(monkeypatch): @@ -343,8 +348,8 @@ def test_azure_region_and_system_service_are_captured(monkeypatch): assert ipv4 == ["20.26.0.0/16"] assert ipv6 == ["2603:1000::/24"] - assert meta["20.26.0.0/16"] == ("uksouth", "AzureCloud") - assert meta["2603:1000::/24"] == ("uksouth", "AzureCloud") + assert meta["20.26.0.0/16"] == ("uksouth", "AzureCloud", "unknown") + assert meta["2603:1000::/24"] == ("uksouth", "AzureCloud", "unknown") def test_azure_global_tag_region_is_labelled_global(monkeypatch): @@ -370,4 +375,4 @@ def test_azure_global_tag_region_is_labelled_global(monkeypatch): "http://example.invalid/az.json", False ) - assert meta["13.64.0.0/16"] == ("global", "AzureCloud") + assert meta["13.64.0.0/16"] == ("global", "AzureCloud", "unknown") diff --git a/tests/test_cloud_service_provider_checks.py b/tests/test_cloud_service_provider_checks.py index 92e412a..070efab 100644 --- a/tests/test_cloud_service_provider_checks.py +++ b/tests/test_cloud_service_provider_checks.py @@ -177,7 +177,7 @@ def test_log_and_write_creates_entry(tmp_path, ctx): # Region and service are what make the match actionable downstream. assert ( out_file.read_text().strip() - == "example.com|34.1.2.3|gcp|europe-west2|Google Cloud|34.0.0.0/8" + == "example.com|34.1.2.3|gcp|europe-west2|Google Cloud|34.0.0.0/8|unknown" ) @@ -408,10 +408,12 @@ def test_handoff_record_carries_region_and_service_end_to_end( perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3"]) line = out_file.read_text().strip() - assert line == "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8" + assert line == "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8|eu-west-2" - domain, ip, provider, region, service, prefix = line.split("|") + domain, ip, provider, region, service, prefix, border = line.split("|") assert (provider, region, service) == ("aws", "eu-west-2", "EC2") + # The border group is the boundary an Elastic IP is allocated from. + assert border == "eu-west-2" def test_each_matched_address_gets_its_own_record(tmp_path, mock_env_manager, ctx): @@ -428,6 +430,52 @@ def test_each_matched_address_gets_its_own_record(tmp_path, mock_env_manager, ct lines = sorted(ln for ln in out_file.read_text().splitlines() if ln) assert lines == [ - "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8", - "example.com|52.1.2.3|aws|us-east-1|AMAZON|52.0.0.0/8", + "example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8|eu-west-2", + "example.com|52.1.2.3|aws|us-east-1|AMAZON|52.0.0.0/8|us-east-1", ] + + +# --------------------------------------------------------------------------- +# Wildcard resolutions in cloud space +# +# A catch-all address belongs to the hosting platform and is in active use, so +# it is the opposite of a claimable target. Unmarked, a single wildcarded zone +# emits one cloud record per enumerated subdomain and buries the real findings. +# --------------------------------------------------------------------------- + + +def test_wildcard_cloud_match_is_marked(tmp_path, mock_env_manager, ctx): + out_file = tmp_path / "csp.txt" + out_file.touch() + mock_env_manager.output_files = {"standard": {"csp": str(out_file)}} + + perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3"], is_wildcard=True) + + line = out_file.read_text().strip() + assert line.startswith("WILDCARD|") + # The record is still complete — marked, not degraded. + assert line == "WILDCARD|example.com|3.1.2.3|aws|eu-west-2|EC2|3.0.0.0/8|eu-west-2" + + +def test_ordinary_cloud_match_is_not_marked(tmp_path, mock_env_manager, ctx): + """A real host's cloud match must keep its existing shape.""" + out_file = tmp_path / "csp.txt" + out_file.touch() + mock_env_manager.output_files = {"standard": {"csp": str(out_file)}} + + perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3"], is_wildcard=False) + + assert not out_file.read_text().startswith("WILDCARD|") + + +def test_wildcard_defaults_to_false_for_existing_callers( + tmp_path, mock_env_manager, ctx +): + """Omitting the flag must not silently mark everything as a wildcard.""" + out_file = tmp_path / "csp.txt" + out_file.touch() + mock_env_manager.output_files = {"standard": {"csp": str(out_file)}} + + perform_csp_checks(ctx, mock_env_manager, ["3.1.2.3"]) + + assert not out_file.read_text().startswith("WILDCARD|") diff --git a/tests/test_run_summary.py b/tests/test_run_summary.py index 149d465..bc9efe5 100644 --- a/tests/test_run_summary.py +++ b/tests/test_run_summary.py @@ -237,3 +237,27 @@ def test_csp_breakdown_shows_region_and_service(tmp_path, capsys): assert "aws us-east-1 AMAZON" in out # The busiest grouping is listed first. assert out.index("eu-west-2") < out.index("us-east-1") + + +def test_wildcard_cloud_records_are_excluded_from_targets(tmp_path, capsys): + """ + Wildcard cloud records are the hosting platform's own addresses. They are + reported for transparency but must not inflate the target counts, or a + single wildcarded zone would dominate the summary. + """ + summary = _make_summary( + tmp_path, + { + "csp": ( + "a.com|1.1.1.1|aws|eu-west-2|EC2|1.1.0.0/16|eu-west-2\n" + "WILDCARD|b.com|2.2.2.2|gcp|europe-west2|Google Cloud|2.2.0.0/16|unknown\n" + "WILDCARD|c.com|2.2.2.2|gcp|europe-west2|Google Cloud|2.2.0.0/16|unknown\n" + ), + }, + ) + summary.display(total_input=3, failed_count=0) + + out = capsys.readouterr().out + assert "AWS: 1" in out + assert "GCP: 0" in out, "wildcard records must not count as GCP targets" + assert "excluded (wildcard) : 2" in out