Require positive confirmation for lightfuzz code-change findings

bbot/core/event/base.py

@@ -958,6 +958,10 @@ def json(self, mode="json"):
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:
             j["web_spider_distance"] = web_spider_distance
+        # kept out of `data` so the snapshot URL doesn't become part of the event's identity
+        archive_url = self.archive_url
+        if archive_url:
+            j["archive_url"] = archive_url
         # scope distance
         j["scope_distance"] = self.scope_distance
         # scan
@@ -1959,7 +1963,8 @@ def _pretty_string(self):
             confidence_str = f"[\033[1m{confidence}\033[0m]"
         else:
             confidence_str = f"[{confidence}]"
-        return f"Severity: [{severity}] Confidence: {confidence_str} {description}"
+        archived_str = "[ARCHIVED] " if self.archive_url else ""
+        return f"{archived_str}Severity: [{severity}] Confidence: {confidence_str} {description}"
 
     def _data_human(self):
         parts = []
@@ -1972,6 +1977,9 @@ def _data_human(self):
         cves = self.data.get("cves", [])
         if cves:
             parts.append(f"[{', '.join(cves)}]")
+        archive_url = self.archive_url
+        if archive_url:
+            parts.append(f"(archived: {archive_url})")
         return " ".join(parts)
 
 

bbot/core/helpers/diff.py

@@ -326,12 +326,18 @@ async def compare(
         else:
             return (False, diff_reasons, reflection, subject_response)
 
-    def _compare_sync(self, subject_response, subject):
-        """CPU-bound comparison work offloaded from the event loop."""
+    @staticmethod
+    def parse_body(text):
+        """Parse a response body into the structure compare_body() expects. Passing raw text works
+        but bypasses the ddiff_filters that mask dynamic content."""
         try:
-            subject_json = xmltodict.parse(subject_response.text)
+            return xmltodict.parse(text)
         except ExpatError:
-            subject_json = subject_response.text.split("\n")
+            return text.split("\n")
+
+    def _compare_sync(self, subject_response, subject):
+        """CPU-bound comparison work offloaded from the event loop."""
+        subject_json = self.parse_body(subject_response.text)
 
         diff_reasons = []
 

bbot/modules/lightfuzz/submodules/serial.py

@@ -6,6 +6,11 @@
 from .base import BaseLightfuzz
 from bbot.errors import HttpCompareError
 
+# enough leading bytes to break every magic header we send (java `AC ED`, dotnet `00 01 00 00`,
+# pickle `80 04`) without disturbing the payload's length or trailing bytes
+MAGIC_HEADER_LENGTH = 4
+HEADER_SCRAMBLE_DELTA = 0x55
+
 
 class _PickleOOB:
     """Pickle-RCE canary: __reduce__ makes the deserializing process resolve
@@ -207,6 +212,27 @@ def payload_language(payload_name):
         """Extract the language family from a payload name (e.g. 'java_base64_string_error' -> 'java')."""
         return payload_name.split("_")[0]
 
+    @staticmethod
+    def corrupt_payload(payload, encoding):
+        """Return a twin of ``payload``: same encoding, length and trailing bytes, scrambled magic
+        header. Parses like the original, deserializes under nothing. None if no twin can be built."""
+        if not payload:
+            return None
+        if encoding == "php_raw":
+            # PHP serialized data leads with a single type character
+            return f"z{payload[1:]}" if payload[0] != "z" else f"q{payload[1:]}"
+        try:
+            data = bytes.fromhex(payload) if encoding == "hex" else base64.b64decode(payload)
+        except ValueError:
+            return None
+        header_length = min(MAGIC_HEADER_LENGTH, len(data))
+        if not header_length:
+            return None
+        corrupted = bytes((b + HEADER_SCRAMBLE_DELTA) % 256 for b in data[:header_length]) + data[header_length:]
+        if encoding == "hex":
+            return corrupted.hex().upper() if payload.isupper() else corrupted.hex()
+        return base64.b64encode(corrupted).decode()
+
     async def confirm_baseline(self, control_payload, cookies):
         """Re-send the control payload to confirm the baseline error state is stable (not transient)."""
         confirmation = await self.standard_probe(self.event.data["type"], cookies, control_payload)
@@ -248,13 +274,13 @@ async def fuzz(self):
 
         # Map each payload set to its control payload for baseline confirmation
         payload_sets = [
-            (base64_serialization_payloads, http_compare_base64, control_payload_base64),
-            (hex_serialization_payloads, http_compare_hex, control_payload_hex),
-            (php_raw_serialization_payloads, http_compare_php_raw, control_payload_php_raw),
+            (base64_serialization_payloads, http_compare_base64, control_payload_base64, "base64"),
+            (hex_serialization_payloads, http_compare_hex, control_payload_hex, "hex"),
+            (php_raw_serialization_payloads, http_compare_php_raw, control_payload_php_raw, "php_raw"),
         ]
 
         # Proceed with payload probes
-        for payload_set, payload_baseline, control_payload in payload_sets:
+        for payload_set, payload_baseline, control_payload, encoding in payload_sets:
             for payload_type, payload in payload_set.items():
                 try:
                     matches_baseline, diff_reasons, reflection, response = await self.compare_probe(
@@ -315,6 +341,21 @@ async def fuzz(self):
                         )
                         continue
 
+                    # a same-shape twin with a scrambled header deserializes under nothing, so if it
+                    # resolves the error too, the value is only being parsed (e.g. as a URL/host)
+                    corrupted_payload = self.corrupt_payload(payload, encoding)
+                    if corrupted_payload is not None:
+                        corrupted_response = await self.standard_probe(
+                            self.event.data["type"], cookies, corrupted_payload
+                        )
+                        corrupted_status = getattr(corrupted_response, "status_code", None)
+                        if corrupted_status == status_code:
+                            self.debug(
+                                f"Corrupted twin of {payload_type} also returned {corrupted_status}, "
+                                "outcome is independent of payload content, skipping"
+                            )
+                            continue
+
                     def get_title(text):
                         soup = self.lightfuzz.helpers.beautifulsoup(text, "html.parser")
                         if soup and soup.title and soup.title.string:

bbot/modules/lightfuzz/submodules/sqli.py

@@ -1,6 +1,13 @@
+import html
+from urllib.parse import quote
+
 from .base import BaseLightfuzz
 from bbot.errors import HttpCompareError
 
+# frameworks disagree on how they spell an escaped quote (Jinja/ASP.NET `'`, Django `'`,
+# PHP `'`, XHTML `'`), and a reflected payload has to be removed in whichever it uses.
+HTML_QUOTE_ENTITIES = ("'", "'", "'", "'")
+
 
 class sqli(BaseLightfuzz):
     """
@@ -13,6 +20,10 @@ class sqli(BaseLightfuzz):
        - Tests quote escape sequence variations
        - Matches against known SQL error patterns
 
+    * Code-change Detection:
+       - Compares the status code of a single-quote probe against a doubled-quote probe
+       - Requires a positive boolean (TRUE/FALSE) content differential to confirm
+
     * Time-based Blind Detection:
        - Uses vendor-specific time delay payloads
        - Confirms delays with statistical analysis
@@ -41,6 +52,19 @@ class sqli(BaseLightfuzz):
         "string not properly terminated",
     ]
 
+    # both halves are the same length and differ by one character, so a reflected copy strips
+    # cleanly and any surviving body difference comes from the query result set
+    BOOLEAN_PROBE_PAIRS = [
+        ("' AND '1'='1", "' AND '1'='2"),
+        (" AND 1=1", " AND 1=2"),
+    ]
+
+    # one vs. two benign characters, mirroring the `'`/`''` pair in length with no SQL meaning
+    BENIGN_CONTROL_SUFFIXES = ("a", "aa")
+
+    # a WAF block or a rate limit says nothing about the query behind the parameter
+    INCONCLUSIVE_STATUS_CODES = (403, 429)
+
     DELAY_PROBE_TEMPLATES = [
         "'||pg_sleep({d})--",
         "' OR (SELECT TRUE FROM pg_sleep({d})) LIMIT 1-- -",
@@ -115,6 +139,102 @@ async def _confirm_code_change(self, probe_value, cookies, initial_status_codes,
 
         return True
 
+    @staticmethod
+    def _strip_payload(text, payload):
+        """Remove reflected copies of a payload from a response body, raw, URL-encoded and HTML-escaped."""
+        escaped = html.escape(payload)
+        variants = [payload, quote(payload), payload.replace(" ", "+"), escaped]
+        variants += [escaped.replace("'", entity) for entity in HTML_QUOTE_ENTITIES]
+        for variant in variants:
+            text = text.replace(variant, "")
+        return text
+
+    async def _probe_body(self, http_compare, payload, cookies):
+        """Send ``payload`` and return its parsed body with reflections stripped, or None when the
+        probe fails or the status is inconclusive."""
+        try:
+            probe = await self.compare_probe(
+                http_compare,
+                self.event.data["type"],
+                payload,
+                cookies,
+                additional_params_populate_empty=True,
+            )
+        except HttpCompareError as e:
+            self.debug(f"Boolean probe [{payload}] failed: {e}")
+            return None
+        if not probe[3]:
+            return None
+        if probe[3].status_code in self.INCONCLUSIVE_STATUS_CODES:
+            self.debug(f"Boolean probe [{payload}] returned {probe[3].status_code}, cannot confirm")
+            return None
+        return http_compare.parse_body(self._strip_payload(probe[3].text, payload))
+
+    async def confirm_boolean_differential(self, http_compare, probe_value, cookies):
+        """Require positive SQL-logic evidence before asserting injection from a status change.
+
+        A WAF signature match or a repacked envelope both produce a bare status flip; only a
+        content differential shows the value reaching a query.
+
+        Returns the confirming ``(true_payload, false_payload)`` pair, or None.
+        """
+        for true_suffix, false_suffix in self.BOOLEAN_PROBE_PAIRS:
+            true_payload = f"{probe_value}{true_suffix}"
+            false_payload = f"{probe_value}{false_suffix}"
+
+            true_body = await self._probe_body(http_compare, true_payload, cookies)
+            if true_body is None:
+                continue
+            false_body = await self._probe_body(http_compare, false_payload, cookies)
+            if false_body is None:
+                continue
+
+            if http_compare.compare_body(true_body, false_body) is not False:
+                self.debug(f"No boolean differential for [{true_suffix}] / [{false_suffix}]")
+                continue
+
+            # an unstable page produces a differential on its own, so the TRUE body must reproduce
+            repeat_body = await self._probe_body(http_compare, true_payload, cookies)
+            if repeat_body is None:
+                continue
+            if http_compare.compare_body(true_body, repeat_body) is False:
+                self.debug("Response body is not deterministic, discarding boolean differential")
+                continue
+
+            self.verbose(f"Boolean differential confirmed for {self.event.url}: [{true_suffix}] vs [{false_suffix}]")
+            return true_payload, false_payload
+        return None
+
+    async def is_quote_specific(self, http_compare, probe_value, cookies, status_codes):
+        """Verify the status flip tracks the quote characters and not the payload's shape.
+
+        If the benign control pair reproduces the same status triplet, the flip tracks value
+        length or envelope validity rather than quoting.
+        """
+        control_codes = []
+        for suffix in self.BENIGN_CONTROL_SUFFIXES:
+            try:
+                control = await self.compare_probe(
+                    http_compare,
+                    self.event.data["type"],
+                    f"{probe_value}{suffix}",
+                    cookies,
+                    additional_params_populate_empty=True,
+                )
+            except HttpCompareError as e:
+                self.debug(f"Quote-specificity control probe failed: {e}")
+                return True
+            if not control[3]:
+                return True
+            control_codes.append(control[3].status_code)
+
+        if (status_codes[0], *control_codes) == status_codes:
+            self.debug(
+                f"Benign control pair reproduced the status triplet {status_codes}, the change is not quote-specific"
+            )
+            return False
+        return True
+
     async def fuzz(self):
         cookies = self.event.data.get("assigned_cookies", {})
         probe_value = self.incoming_probe_value(populate_empty=True)
@@ -187,15 +307,28 @@ async def fuzz(self):
                                 double_single_quote[3].status_code,
                             )
                             confirmed = await self._confirm_code_change(probe_value, cookies, initial_status_codes)
-                            if confirmed:
+                            quote_specific = confirmed and await self.is_quote_specific(
+                                http_compare, probe_value, cookies, initial_status_codes
+                            )
+                            boolean_pair = (
+                                await self.confirm_boolean_differential(http_compare, probe_value, cookies)
+                                if quote_specific
+                                else None
+                            )
+                            if boolean_pair:
                                 self.results.append(
                                     {
                                         "name": "Possible SQL Injection",
                                         "severity": "HIGH",
                                         "confidence": "MEDIUM",
-                                        "description": f"Possible SQL Injection. {self.metadata()} Detection Method: [Single Quote/Two Single Quote, Code Change ({initial_status_codes[0]}->{initial_status_codes[1]}->{initial_status_codes[2]})]",
+                                        "description": f"Possible SQL Injection. {self.metadata()} Detection Method: [Single Quote/Two Single Quote, Code Change ({initial_status_codes[0]}->{initial_status_codes[1]}->{initial_status_codes[2]})] Boolean Confirmation: [{boolean_pair[0]}] vs [{boolean_pair[1]}]",
                                     }
                                 )
+                            elif confirmed and quote_specific:
+                                self.verbose(
+                                    f"Discarding code change {initial_status_codes} for {self.event.url}: "
+                                    "no boolean differential, the value does not reach a query"
+                                )
             else:
                 self.debug("Failed to get responses for both single_quote and double_single_quote")
         except HttpCompareError as e:

bbot/test/test_step_1/test_events.py

@@ -1027,6 +1027,75 @@ async def test_event_web_spider_distance(bbot_scanner):
     assert "spider-max" not in url_event_5.tags
 
 
+@pytest.mark.asyncio
+async def test_event_archived_provenance():
+    """A finding whose evidence is an archived snapshot renders with an [ARCHIVED] marker, so the
+    severity is never read as a claim about the live host."""
+    scan = Scanner()
+    await scan._prep()
+    archived_response = scan.make_event(
+        {
+            "method": "GET",
+            "url": "http://www.evilcorp.com/asdf",
+            "hash": {"header_mmh3": "1", "body_mmh3": "2"},
+            "raw_header": "HTTP/1.1 200 OK\r\n\r\n",
+            "archive_url": "http://web.archive.org/web/20190101000000/http://www.evilcorp.com/asdf",
+        },
+        "HTTP_RESPONSE",
+        parent=scan.root_event,
+        tags=["from-wayback", "archived"],
+    )
+
+    finding = scan.make_event(
+        {
+            "host": "www.evilcorp.com",
+            "description": "test",
+            "severity": "HIGH",
+            "confidence": "HIGH",
+            "name": "Test Finding",
+        },
+        "FINDING",
+        parent=archived_response,
+    )
+    archive_url = "http://web.archive.org/web/20190101000000/http://www.evilcorp.com/asdf"
+    assert finding.archive_url == archive_url
+    assert finding.pretty_string.startswith("[ARCHIVED] Severity: [HIGH]")
+    # output.txt / stdout render data_human, which carries the snapshot URL itself
+    assert finding.data_human.startswith("Severity: [HIGH]")
+    assert finding.data_human.endswith(f"(archived: {archive_url})")
+    # output.json serializes json(), and the snapshot URL must not become part of the finding's identity
+    assert finding.json()["archive_url"] == archive_url
+    assert "archive_url" not in finding.json()["data_json"]
+
+    live_response = scan.make_event(
+        {
+            "method": "GET",
+            "url": "http://www.evilcorp.com/qwerty",
+            "hash": {"header_mmh3": "3", "body_mmh3": "4"},
+            "raw_header": "HTTP/1.1 200 OK\r\n\r\n",
+        },
+        "HTTP_RESPONSE",
+        parent=scan.root_event,
+    )
+    live_finding = scan.make_event(
+        {
+            "host": "www.evilcorp.com",
+            "description": "test",
+            "severity": "HIGH",
+            "confidence": "HIGH",
+            "name": "Live Finding",
+        },
+        "FINDING",
+        parent=live_response,
+    )
+    assert live_finding.archive_url is None
+    assert live_finding.pretty_string.startswith("Severity: [HIGH]")
+    assert live_finding.data_human.startswith("Severity: [HIGH]")
+    assert "archive_url" not in live_finding.json()
+
+    await scan._cleanup()
+
+
 @pytest.mark.asyncio
 async def test_event_closest_host():
     scan = Scanner()