From fd3a82524d0bd89abf707d7d75c91a05e2eb2148 Mon Sep 17 00:00:00 2001 From: Emanuel Flores Date: Mon, 24 Aug 2026 16:38:08 -0700 Subject: [PATCH] fix(py_common): make the FlareSolverr backend usable for JSON endpoints Two bugs made the FlareSolverr backend unable to serve any scraper that fetches a JSON endpoint, so sites behind a Cloudflare managed challenge had no working fallback at all. 1. `params` was silently dropped. FlareSolverr's API has no `params` field and `FlareSolverrBackend.request` only forwarded `json`/`data`, and only for POST. A `scraper.get(url, params={...})` therefore reached the site with no query string. Fold them into the URL instead, before the URL is logged so the request line shows what was actually fetched. 2. The response body was the rendered DOM, not the raw payload. Chrome shows non-HTML documents in its built-in viewer, so FlareSolverr returns `..
{json}
..` and any caller doing `.json()` gets a JSONDecodeError. Unwrap that shell, matching the whole document so a real page containing a `
` is left untouched, and
   unescape the entities Chrome introduces in the text node.

Verified against Babepedia, which is currently behind a managed challenge:
`performer-by-name` went from `JSONDecodeError` (and, once that was fixed, an
empty list from the dropped `params`) to returning correct search results.
`performer-by-url` and other HTML scrapes are byte-for-byte unaffected.
Co-Authored-By: Claude Opus 5 
---
 scrapers/py_common/proxy.py | 43 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/scrapers/py_common/proxy.py b/scrapers/py_common/proxy.py
index f470bf8b0..a646b54ff 100644
--- a/scrapers/py_common/proxy.py
+++ b/scrapers/py_common/proxy.py
@@ -8,6 +8,8 @@
 # caches flaresolverr cookies, userAgent for subsequent requests to same host
 
 import os
+import re
+import html
 import time
 import json
 import urllib.parse
@@ -94,6 +96,9 @@ class FlareSolverrBackend(RequestBackend):
 
   def request(self, method, url, **kwargs):
     if check_flaresolverr(FLARESOLVERR_URL):
+      # FlareSolverr's API has no `params` field, so anything the caller passed
+      # that way has to be folded into the URL or it is silently dropped.
+      url = merge_params(url, kwargs.get("params"))
       log.info(f"[proxy] trying FlareSolverr for {url}")
       # HEAD is not supported
       if method == "head":
@@ -135,6 +140,42 @@ def get_useragent() -> str:
   chrome_ua = requests.get("https://feederbox826.github.io/user-agents/user-agents.json").json()[3]
   return chrome_ua
 
+def merge_params(url: str, params) -> str:
+  """Fold requests-style `params` into `url`, preserving any existing query."""
+  if not params:
+    return url
+  parts = urllib.parse.urlsplit(url)
+  if isinstance(params, (str, bytes)):
+    extra = params.decode() if isinstance(params, bytes) else params
+  else:
+    pairs = params.items() if hasattr(params, "items") else params
+    extra = urllib.parse.urlencode(
+      [(k, v) for k, v in pairs if v is not None], doseq=True
+    )
+  query = f"{parts.query}&{extra}" if parts.query else extra
+  return urllib.parse.urlunsplit(parts._replace(query=query))
+
+# Chrome renders non-HTML documents (JSON, plain text) in its built-in viewer, so
+# FlareSolverr hands back the *rendered DOM* rather than the raw body:
+#   ...
{"the":"real payload"}

+# Any scraper calling .json() on that response dies with a JSONDecodeError, which
+# makes this backend unusable for JSON endpoints. Unwrap only when the document is
+# exactly that viewer shell, so a real page that merely contains a 
 is left alone.
+_CHROME_VIEWER_RE = re.compile(
+  r"\A\s*]*>\s*]*>(?:\s*]*>)*\s*\s*]*>"
+  r"\s*]*>(?P.*)
"
+  # Chrome appends an empty 
 after the 
+  r"(?:\s*]*>\s*
)*\s*\s*\s*\Z",
+  re.DOTALL | re.IGNORECASE,
+)
+
+def unwrap_chrome_viewer(text: str) -> str:
+  match = _CHROME_VIEWER_RE.match(text)
+  if not match:
+    return text
+  log.debug("[proxy] unwrapped Chrome's document viewer from the FlareSolverr response")
+  return html.unescape(match.group("payload"))
+
 def flaresolverr_req(url, method="get", postData=None, proxy=None) -> requests.Response:
   if proxy and "@" in proxy:
     log.warning("[proxy] Ignoring unsupported proxy for FlareSolverr")
@@ -152,7 +193,7 @@ def flaresolverr_req(url, method="get", postData=None, proxy=None) -> requests.R
   solution = response.json().get("solution")
   req_response = requests.Response()
   req_response.status_code = solution.get("status", 200)
-  req_response._content = solution.get("response", "").encode("utf-8")
+  req_response._content = unwrap_chrome_viewer(solution.get("response", "")).encode("utf-8")
   req_response.headers = solution.get("headers", {})
   req_response.url = solution.get("url", "")
   req_response.request = requests.Request(method, req_response.url, headers=req_response.headers).prepare()