diff --git a/agent_reach/channels/web.py b/agent_reach/channels/web.py index 9d10dfe1..1e151b06 100644 --- a/agent_reach/channels/web.py +++ b/agent_reach/channels/web.py @@ -6,6 +6,44 @@ _UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" +# Cloudflare / anti-bot block detection patterns +_CF_BLOCK_PATTERNS = [ + "Just a moment...", + "Checking your browser", + "security verification", + "DDoS protection", + "cf-browser-verify", + "Please turn JavaScript on", + "Attention Required! | Cloudflare", +] + + +def _is_blocked(text: str) -> bool: + """Check whether a response looks like a Cloudflare / anti-bot block page.""" + t = text[:2000] + for pat in _CF_BLOCK_PATTERNS: + if pat.lower() in t.lower(): + return True + return False + + +def _direct_fetch(url: str) -> str | None: + """Attempt a direct HTTP fetch with browser-like headers as fallback.""" + try: + import requests + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + } + resp = requests.get(url, headers=headers, timeout=30, allow_redirects=True) + resp.raise_for_status() + return resp.text + except Exception: + return None + class WebChannel(Channel): name = "web" @@ -22,7 +60,11 @@ def check(self, config=None): return "ok", "通过 Jina Reader 读取任意网页(curl https://r.jina.ai/URL)" def read(self, url: str) -> str: - """通过 Jina Reader 读取网页,返回 Markdown 全文。""" + """通过 Jina Reader 读取网页,返回 Markdown 全文。 + + Falls back to direct HTTP fetch when Jina Reader is blocked + by Cloudflare or similar anti-bot protection. + """ if not url.startswith(("http://", "https://")): url = "https://" + url jina_url = f"https://r.jina.ai/{url}" @@ -31,4 +73,13 @@ def read(self, url: str) -> str: headers={"User-Agent": _UA, "Accept": "text/plain"}, ) with urllib.request.urlopen(req, timeout=30) as resp: - return resp.read().decode("utf-8") + text = resp.read().decode("utf-8") + + # Detect Cloudflare / anti-bot blocks and fall back to a direct fetch. + # If the direct fetch also fails (returns None/empty), keep the + # original Jina response instead of dropping the data. + if _is_blocked(text): + direct = _direct_fetch(url) + return direct or text + + return text