fix: harden web reads and complete P1 cleanup

agent_reach/channels/web.py

@@ -2,9 +2,33 @@
 """Web — any URL via Jina Reader. Always available."""
 
 import urllib.request
+
+from agent_reach.utils.url import normalize_public_http_url
+
 from .base import Channel
 
 _UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+_MAX_RESPONSE_BYTES = 5 * 1024 * 1024
+_ANTIBOT_SCAN_BYTES = 4096
+
+
+def _is_antibot_page(body: bytes) -> bool:
+    """Recognize high-confidence Jina/Cloudflare challenge responses."""
+    sample = body[:_ANTIBOT_SCAN_BYTES].decode("utf-8", errors="ignore").casefold()
+
+    jina_captcha_warning = "warning:" in sample and "requiring captcha" in sample
+    challenge_structure = any(
+        marker in sample
+        for marker in (
+            "title: just a moment...",
+            "## performing security verification",
+            "title: attention required! | cloudflare",
+        )
+    )
+    cloudflare_block = "title: attention required! | cloudflare" in sample and (
+        "ray id" in sample or "/cdn-cgi/challenge-platform/" in sample
+    )
+    return (jina_captcha_warning and challenge_structure) or cloudflare_block
 
 
 class WebChannel(Channel):
@@ -23,12 +47,21 @@ def check(self, config=None):
 
     def read(self, url: str) -> str:
         """通过 Jina Reader 读取网页，返回 Markdown 全文。"""
-        if not url.startswith(("http://", "https://")):
-            url = "https://" + url
+        url = normalize_public_http_url(url)
         jina_url = f"https://r.jina.ai/{url}"
         req = urllib.request.Request(
             jina_url,
             headers={"User-Agent": _UA, "Accept": "text/plain"},
         )
         with urllib.request.urlopen(req, timeout=30) as resp:
-            return resp.read().decode("utf-8")
+            body = resp.read(_MAX_RESPONSE_BYTES + 1)
+        if len(body) > _MAX_RESPONSE_BYTES:
+            raise ValueError(
+                f"Jina Reader response exceeds {_MAX_RESPONSE_BYTES} byte limit"
+            )
+        if _is_antibot_page(body):
+            raise RuntimeError(
+                "Jina Reader 返回了反爬验证页，未获取到目标内容；"
+                "请改用站点专用工具或浏览器读取"
+            )
+        return body.decode("utf-8")

agent_reach/cli.py

@@ -411,10 +411,10 @@ def _cmd_install(args):
 
 
 def _install_skill(force: bool = True):
-    """Install Agent Reach as an agent skill (OpenClaw / Claude Code / .agents)."""
+    """Install Agent Reach as an agent skill for supported agent clients."""
+    import importlib.resources
     import os
     import shutil
-    import importlib.resources
 
     def _is_english_locale(value: str) -> bool:
         normalized = value.strip().lower()
@@ -482,25 +482,28 @@ def _copy_skill_dir(target: str) -> str | None:
             print(f"  Warning: Could not install skill: {e}")
             return None
 
-    # Determine skill install path (priority: .agents > openclaw > claude)
+    # Install into every known skill root that already exists.
     skill_dirs = [
-        os.path.expanduser("~/.agents/skills"),      # Generic agents (priority)
-        os.path.expanduser("~/.openclaw/skills"),    # OpenClaw
-        os.path.expanduser("~/.claude/skills"),      # Claude Code (if exists)
+        (os.path.expanduser("~/.agents/skills"), "Agent"),
+        (os.path.expanduser("~/.config/opencode/skills"), "OpenCode"),
+        (os.path.expanduser("~/.openclaw/skills"), "OpenClaw"),
+        (os.path.expanduser("~/.claude/skills"), "Claude Code"),
     ]
 
     # Insert OPENCLAW_HOME path at the beginning if environment variable is set
     openclaw_home = os.environ.get("OPENCLAW_HOME")
     if openclaw_home:
-        skill_dirs.insert(0, os.path.join(openclaw_home, ".openclaw", "skills"))
+        skill_dirs.insert(
+            0,
+            (os.path.join(openclaw_home, ".openclaw", "skills"), "OpenClaw"),
+        )
 
     installed = False
-    for skill_dir in skill_dirs:
+    for skill_dir, platform_name in skill_dirs:
         if os.path.isdir(skill_dir):
             target = os.path.join(skill_dir, "agent-reach")
             status = _copy_skill_dir(target)
             if status:
-                platform_name = "Agent" if ".agents" in skill_dir else "OpenClaw" if "openclaw" in skill_dir else "Claude Code"
                 if status == "preserved":
                     print(f"Skill already installed for {platform_name}, preserving existing files: {target}")
                 else:
@@ -518,14 +521,18 @@ def _copy_skill_dir(target: str) -> str | None:
             print(f"Skill installed: {target}")
         else:
             print("  -- Could not install agent skill (optional)")
-            print("  -- Tip: install OpenClaw, Claude Code, or create ~/.agents/skills/ manually")
+            print(
+                "  -- Tip: install OpenCode, OpenClaw, Claude Code, "
+                "or create ~/.agents/skills/ manually"
+            )
 
 
 def _uninstall_skill():
     """Remove SKILL.md from all known agent skill directories."""
     import shutil
 
     skill_dirs = [
+        ("~/.config/opencode/skills/agent-reach", "OpenCode"),
         ("~/.openclaw/skills/agent-reach", "OpenClaw"),
         ("~/.claude/skills/agent-reach", "Claude Code"),
         ("~/.agents/skills/agent-reach", "Agent"),
@@ -1649,6 +1656,7 @@ def _cmd_uninstall(args):
 
     # ── 2. Skill files ──
     skill_dirs = [
+        ("~/.config/opencode/skills/agent-reach", "OpenCode"),
         ("~/.openclaw/skills/agent-reach", "OpenClaw"),
         ("~/.claude/skills/agent-reach", "Claude Code"),
         ("~/.agents/skills/agent-reach", "Agent"),

agent_reach/skill/SKILL.md

@@ -19,25 +19,8 @@ description: >
 
   【路由方式】SKILL.md 包含路由表和常用命令，复杂场景需按需阅读对应分类的 references/*.md。
   分类：search / social (小红书/推特/B站/V2EX/Reddit/Facebook/Instagram) / career(LinkedIn) / dev(github) / web(网页/文章/RSS) / video(YouTube/B站/播客) / finance(雪球/股票)。
-triggers:
-  - research: 调研/全网调研/帮我调研/研究一下/research/深入了解
-  - search: 搜/查/找/search/搜索/查一下/帮我搜/看看大家怎么说
-  - social:
-    - 小红书: xiaohongshu/xhs/小红书/红书
-    - Twitter: twitter/推特/x.com/推文
-    - B站: bilibili/b站/哔哩哔哩
-    - V2EX: v2ex
-    - Reddit: reddit
-    - Facebook: facebook/fb/facebook groups
-    - Instagram: instagram/ig
-  - career: 招聘/职位/求职/linkedin/领英/找工作
-  - dev: github/代码/仓库/gh/issue/pr/分支/commit
-  - web: 网页/链接/文章/rss/读一下/打开这个
-  - video: youtube/视频/播客/字幕/小宇宙/转录/yt
-  - finance: 雪球/股票/stock/xueqiu/行情/基金
 metadata:
-  openclaw:
-    homepage: https://github.com/Panniantong/Agent-Reach
+  homepage: https://github.com/Panniantong/Agent-Reach
 ---
 
 # Agent Reach — 互联网能力路由器

agent_reach/skill/SKILL_en.md

@@ -17,8 +17,7 @@ description: >
   internet content); posting/commenting/liking (write operations); platforms
   that already have a dedicated skill installed (prefer that skill).
 metadata:
-  openclaw:
-    homepage: https://github.com/Panniantong/Agent-Reach
+  homepage: https://github.com/Panniantong/Agent-Reach
 ---
 
 # Agent Reach — internet capability router

agent_reach/utils/url.py

@@ -2,8 +2,87 @@
 
 from __future__ import annotations
 
+import ipaddress
+import socket
 from urllib.parse import urlsplit
 
+_BLOCKED_PUBLIC_FETCH_HOSTS = {
+    "home.arpa",
+    "instance-data",
+    "internal",
+    "ip6-localhost",
+    "ip6-loopback",
+    "lan",
+    "local",
+    "localdomain",
+    "localhost",
+    "metadata.google.internal",
+}
+_BLOCKED_PUBLIC_FETCH_SUFFIXES = (
+    ".home.arpa",
+    ".internal",
+    ".lan",
+    ".local",
+    ".localdomain",
+    ".localhost",
+)
+
+
+def _literal_ip_address(
+    host: str,
+) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
+    """Parse canonical and legacy IPv4 literal spellings without DNS."""
+    try:
+        return ipaddress.ip_address(host)
+    except ValueError:
+        pass
+
+    try:
+        packed = socket.inet_aton(host)
+    except OSError:
+        return None
+    return ipaddress.IPv4Address(packed)
+
+
+def normalize_public_http_url(url: str) -> str:
+    """Normalize a URL or reject targets that are not clearly public HTTP(S)."""
+    candidate = str(url or "").strip()
+    if (
+        not candidate
+        or "\\" in candidate
+        or any(
+            character.isspace() or ord(character) < 0x20 or ord(character) == 0x7F
+            for character in candidate
+        )
+    ):
+        raise ValueError("only public HTTP(S) URLs are allowed")
+    if "://" not in candidate:
+        candidate = f"https://{candidate}"
+
+    try:
+        parsed = urlsplit(candidate)
+        host = (parsed.hostname or "").lower().rstrip(".")
+        # Accessing the port rejects malformed or out-of-range authorities.
+        _ = parsed.port
+    except (TypeError, ValueError):
+        raise ValueError("only public HTTP(S) URLs are allowed") from None
+
+    literal_address = _literal_ip_address(host)
+    if (
+        parsed.scheme.lower() not in {"http", "https"}
+        or not host
+        or parsed.username is not None
+        or parsed.password is not None
+        or "%" in host
+        or host in _BLOCKED_PUBLIC_FETCH_HOSTS
+        or host.endswith(_BLOCKED_PUBLIC_FETCH_SUFFIXES)
+        or ("." not in host and literal_address is None)
+        or (literal_address is not None and not literal_address.is_global)
+    ):
+        raise ValueError("only public HTTP(S) URLs are allowed")
+
+    return parsed.geturl()
+
 
 def domain_matches(host: str, *domains: str) -> bool:
     """Match a hostname/cookie domain exactly or as a real subdomain."""

docs/README_ja.md

@@ -64,10 +64,8 @@ Update Agent Reach: https://raw.githubusercontent.com/Panniantong/agent-reach/ma
 | 🌐 **Web** | 閲覧 | 設定不要 | 任意のURL → クリーンなMarkdown（[Jina Reader](https://github.com/jina-ai/reader) ⭐9.8K） |
 | 🐦 **Twitter/X** | 閲覧・検索 | 設定不要 / Cookie | 単一ツイートはすぐに閲覧可能。Cookieで検索、タイムライン、投稿が解放（[twitter-cli](https://github.com/public-clis/twitter-cli)） |
 | 📕 **小紅書** | 閲覧・検索・コメント | OpenCLI / Cookie | OpenCLI はユーザー管理の既存 Chrome セッションだけを使用。MCP/旧ツールは Cookie-Editor を使用 |
-| 🎵 **抖音** | 動画解析・ウォーターマークなしダウンロード | mcporter | [douyin-mcp-server](https://github.com/yzfly/douyin-mcp-server)、ログイン不要 |
 | 💼 **LinkedIn** | Jina Reader（公開ページ） | プロフィール、企業、求人検索 | エージェントに「LinkedInの設定を手伝って」と伝えてください |
 | 💬 **WeChat記事** | 検索 + 閲覧 | 設定不要 | WeChat公式アカウント記事の検索+閲覧（完全Markdown）（[Exa](https://exa.ai) + [Camoufox](https://github.com/daijro/camoufox)（オプション）） |
-| 📰 **Weibo** | トレンド・検索・フィード・コメント | 設定不要 | ホット検索、コンテンツ/ユーザー/トピック検索、フィード、コメント（[mcp-server-weibo](https://github.com/Panniantong/mcp-server-weibo)） |
 | 💻 **V2EX** | 人気トピック・ノードトピック・トピック詳細+返信・ユーザープロフィール | 設定不要 | 公開JSON API、認証不要。技術コミュニティのコンテンツに最適 |
 | 📈 **雪球（Xueqiu）** | 株価・検索・人気投稿・人気銘柄 | 設定不要 | 公開APIで自動セッションCookie、ログイン不要 |
 | 🎙️ **小宇宙Podcast** | 文字起こし | 無料APIキー | Podcast音声 → Groq Whisper（無料）による完全テキスト文字起こし |
@@ -206,7 +204,6 @@ channels/
 ├── bilibili.py     → bili-cli ▸ OpenCLI ▸ 検索 API（yt-dlp は 412 制限により退役）
 ├── reddit.py       → OpenCLI ▸ rdt-cli（ログイン状態が必要）
 ├── xiaohongshu.py  → OpenCLI ▸ xiaohongshu-mcp ▸ xhs-cli
-├── douyin.py       → mcporter MCP    ← 他の抖音ツールに差し替え可能…
 ├── linkedin.py     → linkedin-mcp    ← LinkedIn APIに差し替え可能…
 ├── rss.py          → feedparser      ← atomaなどに差し替え可能…
 ├── exa_search.py   → mcporter MCP    ← Tavily、SerpAPIなどに差し替え可能…
@@ -227,10 +224,8 @@ channels/
 | GitHub | [gh CLI](https://cli.github.com) | 公式ツール、認証後フルAPI |
 | RSS閲覧 | [feedparser](https://github.com/kurtmckee/feedparser) | Pythonエコシステムの標準、⭐2.3K |
 | 小紅書 | [OpenCLI](https://github.com/jackwener/opencli)（デスクトップ）▸ [xiaohongshu-mcp](https://github.com/xpzouying/xiaohongshu-mcp)（サーバー）▸ xhs-cli | OpenCLI は既存のユーザー管理セッションのみ使用。その他は Cookie-Editor で手動設定 |
-| 抖音 | [douyin-mcp-server](https://github.com/yzfly/douyin-mcp-server) | MCPサーバー、ログイン不要、動画解析 + ウォーターマークなしダウンロード |
 | LinkedIn | [linkedin-scraper-mcp](https://github.com/stickerdaniel/linkedin-mcp-server) | ⭐900+、MCPサーバー、ブラウザ自動化 |
 | WeChat記事 | [Exa](https://exa.ai)（検索+閲覧）+ [Camoufox](https://github.com/daijro/camoufox)（オプション） | ゼロ設定で検索+全文閲覧、Camoufoxでオプション強化 |
-| Weibo | `mcporter` | `mcporter call 'weibo.get_trendings(limit: 10)'` |
 | 小宇宙Podcast | `transcribe.sh` | `bash ~/.agent-reach/tools/xiaoyuzhou/transcribe.sh <URL>` |
 
 > 📌 これらは*現在*の選択です。気に入らなければファイルを差し替えるだけ。それがスキャフォールディングの要点です。
@@ -293,17 +288,11 @@ Agent Reach はtwitter-cliを使用し、Cookie認証でTwitterにアクセス
 Agent Reach は小紅書へのログインを代行せず、ブラウザ Cookie も読み取りません。OpenCLI はユーザーが既に所有・管理している Chrome セッションだけを使用します。既存セッションがない場合は自動ログインせず、Cookie-Editor で手動エクスポートして xiaohongshu-mcp または旧ツールを設定してください。`agent-reach configure xhs-cookies` は OpenCLI / Chrome に Cookie を注入しません。
 </details>
 
-<details>
-<summary><strong>AIエージェントで抖音の動画を解析するには？</strong></summary>
-
-douyin-mcp-serverをインストールすれば、`mcporter call 'douyin.parse_douyin_video_info(share_link: "share_url")'` で動画情報を解析し、ウォーターマークなしのダウンロードリンクを取得できます。ログイン不要 — 抖音のリンクを共有するだけ。詳細は https://github.com/yzfly/douyin-mcp-server を参照。
-</details>
-
 ---
 
 ## クレジット
 
-[twitter-cli](https://github.com/public-clis/twitter-cli) · [rdt-cli](https://github.com/public-clis/rdt-cli) · [xhs-cli](https://github.com/jackwener/xiaohongshu-cli) · [Jina Reader](https://github.com/jina-ai/reader) · [yt-dlp](https://github.com/yt-dlp/yt-dlp) · [Exa](https://exa.ai) · [feedparser](https://github.com/kurtmckee/feedparser) · [douyin-mcp-server](https://github.com/yzfly/douyin-mcp-server) · [linkedin-scraper-mcp](https://github.com/stickerdaniel/linkedin-mcp-server)
+[twitter-cli](https://github.com/public-clis/twitter-cli) · [rdt-cli](https://github.com/public-clis/rdt-cli) · [xhs-cli](https://github.com/jackwener/xiaohongshu-cli) · [Jina Reader](https://github.com/jina-ai/reader) · [yt-dlp](https://github.com/yt-dlp/yt-dlp) · [Exa](https://exa.ai) · [feedparser](https://github.com/kurtmckee/feedparser) · [linkedin-scraper-mcp](https://github.com/stickerdaniel/linkedin-mcp-server)
 
 ## お問い合わせ