diff --git a/agent_reach/channels/v2ex.py b/agent_reach/channels/v2ex.py index b6032522..6e7884fe 100644 --- a/agent_reach/channels/v2ex.py +++ b/agent_reach/channels/v2ex.py @@ -2,22 +2,108 @@ """V2EX — public API channel for topics, nodes, users, and replies.""" import json +import shutil +import ssl +import subprocess import urllib.request from typing import Any +from agent_reach.utils.process import utf8_subprocess_env from agent_reach.utils.text import scrub_url_credentials from .base import Channel _UA = "agent-reach/1.0" _TIMEOUT = 10 +_MAX_RESPONSE_BYTES = 1024 * 1024 -def _get_json(url: str) -> Any: - """Fetch *url* and return parsed JSON. Raises on HTTP/network errors.""" +def _get_json_with_urllib(url: str) -> Any: + """Fetch JSON with Python's standard HTTP stack.""" req = urllib.request.Request(url, headers={"User-Agent": _UA}) with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp: - return json.loads(resp.read().decode("utf-8")) + raw = resp.read(_MAX_RESPONSE_BYTES + 1) + if len(raw) > _MAX_RESPONSE_BYTES: + raise ValueError("V2EX API response exceeds the 1 MiB safety limit") + return json.loads(raw.decode("utf-8")) + + +def _is_unexpected_tls_eof(error: BaseException) -> bool: + """Return whether an exception chain contains the retryable TLS EOF.""" + pending: list[BaseException] = [error] + seen: set[int] = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + text = str(current).casefold() + if ( + "unexpected_eof_while_reading" in text + or "eof occurred in violation of protocol" in text + ): + return True + for nested in ( + getattr(current, "reason", None), + current.__cause__, + current.__context__, + ): + if isinstance(nested, BaseException): + pending.append(nested) + return False + + +def _get_json_with_curl(url: str) -> Any: + """Fetch bounded JSON with the OS curl TLS stack.""" + curl = shutil.which("curl") + if not curl: + raise RuntimeError("curl is unavailable for the V2EX TLS fallback") + + command = [ + curl, + "--fail", + "--silent", + "--show-error", + "--location", + "--connect-timeout", + "5", + "--max-time", + str(_TIMEOUT), + "--max-filesize", + str(_MAX_RESPONSE_BYTES), + "--header", + f"User-Agent: {_UA}", + "--url", + url, + ] + try: + result = subprocess.run( + command, + capture_output=True, + encoding="utf-8", + errors="replace", + timeout=_TIMEOUT + 2, + env=utf8_subprocess_env(), + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise RuntimeError("curl could not complete the V2EX TLS fallback") from exc + if result.returncode != 0: + raise RuntimeError("curl could not complete the V2EX TLS fallback") + if len(result.stdout.encode("utf-8")) > _MAX_RESPONSE_BYTES: + raise ValueError("V2EX API response exceeds the 1 MiB safety limit") + return json.loads(result.stdout) + + +def _get_json(url: str) -> Any: + """Fetch JSON, retrying only Python's known TLS EOF via native curl.""" + try: + return _get_json_with_urllib(url) + except Exception as exc: + if isinstance(exc, ssl.SSLCertVerificationError): + raise + if not _is_unexpected_tls_eof(exc): + raise + return _get_json_with_curl(url) class V2EXChannel(Channel): diff --git a/tests/test_v2ex_channel.py b/tests/test_v2ex_channel.py index 2379ed62..cdf76b8f 100644 --- a/tests/test_v2ex_channel.py +++ b/tests/test_v2ex_channel.py @@ -11,12 +11,17 @@ reddit (#364) and xueqiu (#365). """ +import json +import ssl +import subprocess from unittest.mock import patch +from urllib.error import URLError + +import pytest from agent_reach.channels import v2ex as v2 from agent_reach.channels.v2ex import V2EXChannel - # --- can_handle --- def test_can_handle_matches_v2ex_hosts(): @@ -47,6 +52,70 @@ def test_check_warn_on_exception_clears_backend(): assert ch.active_backend is None +def test_get_json_retries_unexpected_tls_eof_with_bounded_curl(): + payload = [{"id": 1}] + tls_error = URLError( + ssl.SSLError( + "[SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol" + ) + ) + + with patch.object(v2, "_get_json_with_urllib", side_effect=tls_error), patch.object( + v2.shutil, "which", return_value="C:/Windows/System32/curl.exe" + ), patch.object( + v2.subprocess, + "run", + return_value=subprocess.CompletedProcess( + ["curl"], 0, json.dumps(payload), "" + ), + ) as run: + assert v2._get_json("https://www.v2ex.com/api/topics/hot.json") == payload + + command = run.call_args.args[0] + assert command[0] == "C:/Windows/System32/curl.exe" + assert "--fail" in command + assert "--max-time" in command + assert "--max-filesize" in command + assert command[-2:] == [ + "--url", + "https://www.v2ex.com/api/topics/hot.json", + ] + assert run.call_args.kwargs["timeout"] == v2._TIMEOUT + 2 + + +def test_get_json_does_not_hide_certificate_verification_failures(): + certificate_error = ssl.SSLCertVerificationError( + "certificate verify failed" + ) + + with patch.object( + v2, "_get_json_with_urllib", side_effect=certificate_error + ), patch.object(v2.subprocess, "run") as run: + with pytest.raises(ssl.SSLCertVerificationError): + v2._get_json("https://www.v2ex.com/api/topics/hot.json") + + run.assert_not_called() + + +def test_check_is_healthy_when_native_curl_recovers_tls_eof(): + ch = V2EXChannel() + tls_error = ssl.SSLError( + "[SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol" + ) + + with patch.object(v2, "_get_json_with_urllib", side_effect=tls_error), patch.object( + v2.shutil, "which", return_value="/usr/bin/curl" + ), patch.object( + v2.subprocess, + "run", + return_value=subprocess.CompletedProcess(["curl"], 0, "[]", ""), + ): + status, _message = ch.check() + + assert status == "ok" + assert ch.active_backend == ch.backends[0] + + # --- get_hot_topics / get_node_topics --- def test_get_hot_topics_maps_node_and_truncates_content():