From c955b411b38eebd29e363144ba7f24f3011a170b Mon Sep 17 00:00:00 2001 From: lym12321 <41183806+lym12321@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:08:55 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E6=8E=A5=E5=85=A5=E6=B5=8F=E8=A7=88?= =?UTF-8?q?=E5=99=A8=E6=8A=93=E5=8F=96=E5=B9=B6=E4=B8=8A=E6=8A=A5=E9=99=84?= =?UTF-8?q?=E4=BB=B6=E4=B8=8B=E8=BD=BD=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/pyproject.toml | 1 + crawler/src/crawler/crawl/attachments.py | 115 ++ crawler/src/crawler/crawl/attachments_test.py | 157 +++ crawler/src/crawler/crawl/browser.py | 530 ++++++++ crawler/src/crawler/crawl/browser_test.py | 1183 +++++++++++++++++ crawler/src/crawler/crawl/contract.py | 10 +- crawler/src/crawler/crawl/fetch.py | 119 +- crawler/src/crawler/crawl/fetch_test.py | 90 +- crawler/src/crawler/crawl/rounds.py | 123 +- crawler/src/crawler/crawl/rounds_test.py | 170 ++- crawler/src/crawler/crawl/website.py | 3 +- docs/api.md | 8 +- docs/data-model.md | 2 +- docs/design.md | 4 +- docs/development.md | 20 +- web/db/migrations/0001_init.sql | 1 + web/db/types.ts | 1 + web/openapi/crawler.json | 18 +- web/server/modules/ingest/routes.test.ts | 311 ++++- web/server/modules/ingest/routes.ts | 14 +- web/server/modules/ingest/service.ts | 137 +- 21 files changed, 2850 insertions(+), 167 deletions(-) create mode 100644 crawler/src/crawler/crawl/attachments.py create mode 100644 crawler/src/crawler/crawl/attachments_test.py create mode 100644 crawler/src/crawler/crawl/browser.py create mode 100644 crawler/src/crawler/crawl/browser_test.py diff --git a/crawler/pyproject.toml b/crawler/pyproject.toml index c640430..ac2d3b4 100644 --- a/crawler/pyproject.toml +++ b/crawler/pyproject.toml @@ -6,6 +6,7 @@ requires-python = ">=3.14" dependencies = [ "httpx>=0.28", "lxml>=6.1.3", + "playwright>=1.64.1", "pydantic>=2.14.1", ] diff --git a/crawler/src/crawler/crawl/attachments.py b/crawler/src/crawler/crawl/attachments.py new file mode 100644 index 0000000..2dbcc32 --- /dev/null +++ b/crawler/src/crawler/crawl/attachments.py @@ -0,0 +1,115 @@ +import logging +import re +from dataclasses import dataclass +from html.parser import HTMLParser + +from crawler.crawl.contract import AttachmentInfo +from crawler.crawl.fetch import FetchError, PageSession, Response + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Attachment: + info: AttachmentInfo + data: bytes | None + + +class _DownloadPage(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.parts: list[str] = [] + self.hidden = 0 + self.office = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "html": + self.office = any( + (name == "xmlns" or name.startswith("xmlns:")) + and value is not None + and value.startswith("urn:schemas-microsoft-com:office:") + for name, value in attrs + ) + if tag in {"title", "script", "style"}: + self.hidden += 1 + + def handle_endtag(self, tag: str) -> None: + if tag in {"title", "script", "style"}: + self.hidden = max(0, self.hidden - 1) + + def handle_data(self, data: str) -> None: + if not self.hidden: + self.parts.append(data) + + +def _download_error(response: Response) -> str | None: + status = f"HTTP {response.status}" if not 200 <= response.status < 300 else None + # Some download endpoints label Office and PDF files as HTML. + if status is None and response.body.startswith((b"%PDF-", b"\xd0\xcf\x11\xe0", b"PK\x03\x04")): + return None + content_type = response.content_type or "" + charset = re.search(r"charset\s*=\s*['\"]?([\w.-]+)", content_type, re.IGNORECASE) + meta = re.search( + rb"]*\bcharset\s*=\s*['\"]?([\w.-]+)", response.body[:1024], re.IGNORECASE + ) + if response.body.startswith(b"\xef\xbb\xbf"): + encoding = "utf-8-sig" + elif response.body.startswith((b"\xff\xfe", b"\xfe\xff")): + encoding = "utf-16" + elif charset: + encoding = charset[1] + elif meta: + encoding = meta[1].decode("ascii") + else: + encoding = "utf-8-sig" + try: + prefix = response.body[:1024].decode(encoding, errors="replace") + except LookupError: + encoding = "utf-8-sig" + prefix = response.body[:1024].decode(encoding, errors="replace") + html = content_type.split(";")[0].strip().lower() in {"text/html", "application/xhtml+xml"} + html = ( + html + or re.match( + r"<(?:!doctype\s+html|html|head|body|title|meta|script|style|form|div|p|span|pre|table|h[1-6])" + r"(?:\s|/?>)", + prefix.lstrip(), + re.IGNORECASE, + ) + is not None + ) + if not html: + return status or ("附件内容为空" if not response.body else None) + + page = _DownloadPage() + page.feed(response.body.decode(encoding, errors="replace")) + if status is None and page.office: + return None + text = " ".join(" ".join(page.parts).split()) + if "验证码" in text or "captcha" in text.lower(): + return "下载附件需要验证码" + # A whole notice or login page is not a useful attachment failure message. + detail = text if 0 < len(text) <= 1000 else "附件地址未返回文件" + return detail if status is None else f"{status}: {detail}" + + +async def download_attachment( + pages: PageSession, url: str, *, name: str, page_url: str +) -> Attachment: + content_type = "application/octet-stream" + data: bytes | None = None + try: + response = await pages.download(url, page_url=page_url) + except FetchError as failure: + error = failure.failure + else: + content_type = response.content_type or content_type + error = _download_error(response) + if error is None: + data = response.body + if error is not None: + logger.warning("Attachment download failed", extra={"url": url}) + return Attachment( + AttachmentInfo(name=name, url=url, content_type=content_type, text="", error=error), + data, + ) diff --git a/crawler/src/crawler/crawl/attachments_test.py b/crawler/src/crawler/crawl/attachments_test.py new file mode 100644 index 0000000..1b27116 --- /dev/null +++ b/crawler/src/crawler/crawl/attachments_test.py @@ -0,0 +1,157 @@ +import asyncio + +import httpx +import pytest + +from crawler.crawl.attachments import download_attachment +from crawler.crawl.fetch import Fetcher + +URL = "https://school.test/download.jsp?file=1" +PAGE_URL = "https://school.test/notice.htm" + + +@pytest.mark.parametrize( + ("body", "content_type", "status", "error"), + [ + ( + '
请输入验证码下载附件
', + "text/html;charset=UTF-8", + 200, + "下载附件需要验证码", + ), + ("
获取文件错误
", "text/html;charset=UTF-8", 200, "获取文件错误"), + ("

附件已删除

", "text/html;charset=gb2312", 200, "附件已删除"), + ( + '

文件已删除

', + "text/html;charset=gb2312", + 200, + "文件已删除", + ), + ( + "

文件已删除

".encode("utf-8-sig"), + "text/html;charset=gb2312", + 200, + "文件已删除", + ), + ( + ( + '请输入验证码下载附件' + ).encode("gb2312"), + "text/html", + 200, + "下载附件需要验证码", + ), + ( + ( + '' + "

文件已删除

" + ).encode("gb2312"), + "text/html", + 200, + "文件已删除", + ), + ( + "

请输入验证码下载附件

".encode("utf-16"), + None, + 200, + "下载附件需要验证码", + ), + ( + "下载请输入验证码下载附件", + "text/html;charset=UTF-8", + 200, + "下载附件需要验证码", + ), + ( + "下载

文件已删除

", + "text/html;charset=UTF-8", + 200, + "文件已删除", + ), + ("

文件不存在

", "text/html", 404, "HTTP 404: 文件不存在"), + ("", None, 200, "附件地址未返回文件"), + ( + "文件已删除", + "application/octet-stream", + 200, + "文件已删除", + ), + ("\ufeff\n
获取文件错误
", None, 200, "获取文件错误"), + ("", "application/pdf", 200, "附件内容为空"), + ("denied", "text/plain", 403, "HTTP 403"), + ], +) +def test_download_errors_keep_the_link_and_never_keep_an_error_page_as_a_file( + body: str | bytes, content_type: str | None, status: int, error: str +) -> None: + encoding = "gb2312" if content_type and "gb2312" in content_type else "utf-8" + content = body.encode(encoding) if isinstance(body, str) else body + headers = {"content-type": content_type} if content_type else {} + + async def main() -> None: + transport = httpx.MockTransport( + lambda _: httpx.Response(status, content=content, headers=headers) + ) + async with Fetcher(transport) as fetcher, fetcher.session(False) as pages: + attachment = await download_attachment(pages, URL, name="申请表.doc", page_url=PAGE_URL) + assert attachment.info.url == URL + assert attachment.info.name == "申请表.doc" + assert attachment.info.text == "" + assert attachment.info.error == error + assert attachment.data is None + + asyncio.run(main()) + + +@pytest.mark.parametrize( + ("body", "content_type"), + [ + (b"%PDF-1.7\n", "text/html"), + (b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", "text/html"), + (b"PK\x03\x04", "text/html"), + ( + '' + '' + '申请表' + "
".encode(), + "application/vnd.ms-excel", + ), + ( + '' + "申请表" + "
申请表
".encode(), + "application/vnd.ms-excel", + ), + ], +) +def test_downloads_keep_binary_and_office_files_after_redirects( + body: bytes, content_type: str +) -> None: + async def main() -> None: + def handle(request: httpx.Request) -> httpx.Response: + if request.url.path == "/download.jsp": + return httpx.Response(302, headers={"location": "https://files.test/form"}) + return httpx.Response(200, content=body, headers={"content-type": content_type}) + + async with Fetcher(httpx.MockTransport(handle)) as fetcher, fetcher.session(False) as pages: + attachment = await download_attachment(pages, URL, name="申请表", page_url=PAGE_URL) + assert attachment.info.url == URL + assert attachment.info.error is None + assert attachment.data == body + + asyncio.run(main()) + + +def test_a_transport_failure_is_an_attachment_error() -> None: + async def main() -> None: + def handle(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("timed out", request=request) + + async with Fetcher(httpx.MockTransport(handle)) as fetcher, fetcher.session(False) as pages: + attachment = await download_attachment(pages, URL, name="申请表", page_url=PAGE_URL) + assert attachment.info.url == URL + assert attachment.info.error == "timeout" + assert attachment.data is None + + asyncio.run(main()) diff --git a/crawler/src/crawler/crawl/browser.py b/crawler/src/crawler/crawl/browser.py new file mode 100644 index 0000000..63ff05c --- /dev/null +++ b/crawler/src/crawler/crawl/browser.py @@ -0,0 +1,530 @@ +import asyncio +import logging +import time +from collections.abc import Awaitable, Callable +from typing import Protocol +from urllib.parse import urljoin, urlsplit + +from playwright.async_api import ( + Browser as PlaywrightBrowser, +) +from playwright.async_api import ( + BrowserContext, + Download, + Frame, + Page, + Playwright, + Route, + async_playwright, +) +from playwright.async_api import ( + Error as PlaywrightError, +) +from playwright.async_api import ( + Request as PlaywrightRequest, +) +from playwright.async_api import ( + Response as PlaywrightResponse, +) +from playwright.async_api import ( + TimeoutError as PlaywrightTimeoutError, +) +from pydantic import BaseModel, Field, JsonValue, TypeAdapter, ValidationError + +from crawler.crawl.fetch import ( + MAX_REDIRECTS, + TIMEOUT, + USER_AGENT, + FetchError, + Permit, + Response, + site_key, +) + +PAGE_TIMEOUT = 300.0 +PAGE_LIMIT = 2 +_REDIRECTS = {301, 302, 303, 307, 308} +_BLOCKED = {"image", "font", "media"} +_CHALLENGES = {202, 412} +_PARAMS = TypeAdapter(dict[str, JsonValue]) +_logger = logging.getLogger(__name__) + + +class _Version(BaseModel): + user_agent: str = Field(alias="userAgent") + + +class _Request(BaseModel): + url: str + + +class _Paused(BaseModel): + request_id: str = Field(alias="requestId") + request: _Request + frame_id: str = Field(alias="frameId") + redirected_request_id: str | None = Field(default=None, alias="redirectedRequestId") + + +class _Frame(BaseModel): + id: str + + +class _FrameTree(BaseModel): + frame: _Frame + + +class _Frames(BaseModel): + frame_tree: _FrameTree = Field(alias="frameTree") + + +# Playwright leaves CDPSession.send's dictionaries untyped; validate its JSON at this boundary. +class _Commands(Protocol): + async def send(self, method: str, params: dict[str, JsonValue] | None = None) -> object: ... + + def on(self, event: str, f: Callable[[object], None]) -> None: ... + + async def detach(self) -> None: ... + + +async def _command( + connection: _Commands, method: str, params: dict[str, JsonValue] | None = None +) -> dict[str, JsonValue]: + return _PARAMS.validate_python(await connection.send(method, params)) + + +def _source_error(error: PlaywrightError, url: str) -> FetchError | None: + if isinstance(error, PlaywrightTimeoutError): + return FetchError("timeout", url) + message = str(error).split("\n", 1)[0] + if "Download is starting" in message: + return FetchError("unreachable", url) + if "net::ERR_TOO_MANY_REDIRECTS" in message: + return FetchError("too_many_redirects", url) + if "net::ERR_TIMED_OUT" in message or "Request timed out" in message: + return FetchError("timeout", url) + if any( + marker in message + for marker in ("net::ERR_", "ECONN", "ENOTFOUND", "EAI_AGAIN", "socket hang up") + ): + return FetchError("unreachable", url) + if message.startswith( + ( + "APIRequestContext.get: self-signed certificate", + "APIRequestContext.get: certificate has expired", + "APIRequestContext.get: unable to verify the first certificate", + "APIRequestContext.get: Hostname/IP does not match certificate's altnames:", + "APIRequestContext.get: failed to decompress 'gzip' encoding:", + ) + ): + return FetchError("unreachable", url) + return None + + +class Browser: + def __init__(self, permit: Callable[[str], Awaitable[Permit]]) -> None: + self.permit = permit + self.slots = asyncio.Semaphore(PAGE_LIMIT) + self._lock = asyncio.Lock() + self._playwright: Playwright | None = None + self._browser: PlaywrightBrowser | None = None + self._user_agent = "" + self.sessions: set[BrowserSession] = set() + + async def _start(self) -> PlaywrightBrowser: + async with self._lock: + if self._browser is not None: + return self._browser + playwright = await async_playwright().start() + try: + browser = await playwright.chromium.launch( + headless=True, args=["--disable-blink-features=AutomationControlled"] + ) + connection: _Commands = await browser.new_browser_cdp_session() + try: + version = _Version.model_validate( + await _command(connection, "Browser.getVersion") + ) + finally: + await connection.detach() + except BaseException: + await playwright.stop() + raise + self._user_agent = ( + version.user_agent.replace("HeadlessChrome", "Chrome") + " " + USER_AGENT + ) + self._playwright = playwright + self._browser = browser + return browser + + async def session(self) -> BrowserSession: + browser = await self._start() + creating = asyncio.create_task( + browser.new_context( + user_agent=self._user_agent, service_workers="block", accept_downloads=False + ) + ) + try: + # Cancelling a protocol call does not prevent the browser from creating its resource. + await asyncio.wait((creating,)) + context = creating.result() + session = BrowserSession(self, context) + await session.configure() + except BaseException: + context = await creating + await context.close() + raise + self.sessions.add(session) + return session + + async def close(self) -> None: + try: + for session in tuple(self.sessions): + await session.close() + finally: + try: + if self._browser is not None: + await self._browser.close() + self._browser = None + finally: + if self._playwright is not None: + await self._playwright.stop() + self._playwright = None + + +class BrowserSession: + def __init__(self, owner: Browser, context: BrowserContext) -> None: + self._owner = owner + self._context = context + self._pages: set[Page] = set() + self._popup_tasks: set[asyncio.Task[None]] = set() + self._closed = False + + async def configure(self) -> None: + await self._context.route("**/*", self._route) + self._context.on("page", self._new_page) + + def _new_page(self, page: Page) -> None: + task = asyncio.create_task(self._popup(page)) + self._popup_tasks.add(task) + + async def _popup(self, page: Page) -> None: + if await page.opener() is not None: + await page.close() + + async def _route(self, route: Route) -> None: + request = route.request + if request.resource_type in _BLOCKED: + await route.abort() + return + try: + allowed = request.frame.page in self._pages + except PlaywrightError as error: + if "Frame for this navigation request is not available" not in str(error): + raise + allowed = False + if not allowed: + await route.abort() + return + await route.continue_() + + async def get(self, url: str) -> Response: + async with self._owner.slots: + page: Page | None = None + creating: asyncio.Task[Page] | None = None + request: _Page | None = None + started = time.monotonic() + try: + async with asyncio.timeout(PAGE_TIMEOUT): + creating = asyncio.create_task(self._context.new_page()) + await asyncio.wait((creating,)) + page = creating.result() + self._pages.add(page) + request = _Page(page, self._context, self._owner.permit, url) + response = await request.run() + _logger.info( + "Browser fetched %s status=%s elapsed=%.3fs", + response.url, + response.status, + time.monotonic() - started, + ) + return response + except TimeoutError as error: + raise FetchError("timeout", url) from error + finally: + if request is not None: + await request.close() + elif creating is not None: + page = await creating + await page.close() + if page is not None: + self._pages.discard(page) + + async def download(self, url: str, *, page_url: str) -> Response: + for _ in range(MAX_REDIRECTS + 1): + permit = await self._owner.permit(url) + try: + response = await self._context.request.get( + url, + headers={"Referer": page_url}, + timeout=TIMEOUT * 1000, + max_redirects=0, + max_retries=0, + fail_on_status_code=False, + ) + try: + location = response.headers.get("location") + if response.status in _REDIRECTS and location is not None: + url = urljoin(response.url, location) + continue + return Response( + response.url, + response.status, + response.headers.get("content-type"), + await response.body(), + ) + finally: + await response.dispose() + except PlaywrightError as error: + failure = _source_error(error, url) + if failure is not None: + raise failure from error + raise + finally: + permit.release() + raise FetchError("too_many_redirects", url) + + async def close(self) -> None: + if self._closed: + return + self._closed = True + try: + await self._context.close() + finally: + try: + if self._popup_tasks: + await asyncio.gather(*self._popup_tasks) + finally: + self._owner.sessions.discard(self) + + +class _Page: + def __init__( + self, + page: Page, + context: BrowserContext, + permit: Callable[[str], Awaitable[Permit]], + url: str, + ) -> None: + self._context = context + self._permit = permit + self._page = page + self._url = url + self._connection: _Commands | None = None + self._frame_id = "" + self._tasks: set[asyncio.Task[None]] = set() + self._gate = asyncio.Lock() + self._current_permit: Permit | None = None + self._site: str | None = None + self._redirects: dict[str, int] = {} + self._changed = asyncio.Event() + self._failure: asyncio.Future[BaseException] = asyncio.get_running_loop().create_future() + self._main: PlaywrightResponse | None = None + self._current_request: PlaywrightRequest | None = None + self._generation = 0 + self._committed_generation = -1 + self._loaded_generation = -1 + self._document_pending = False + self._closing = False + + def _spawn(self, operation: Awaitable[None]) -> None: + task = asyncio.ensure_future(operation) + self._tasks.add(task) + task.add_done_callback(self._done) + + def _done(self, task: asyncio.Task[None]) -> None: + self._tasks.discard(task) + if not task.cancelled(): + error = task.exception() + if error is not None: + self._fail(error) + self._changed.set() + + def _fail(self, error: BaseException) -> None: + if not self._closing and not self._failure.done(): + self._failure.set_result(error) + + async def _send( + self, method: str, params: dict[str, JsonValue] | None = None + ) -> dict[str, JsonValue]: + if self._connection is None: + raise RuntimeError("CDP connection has not started") + return await _command(self._connection, method, params) + + async def _configure(self) -> None: + frames = _Frames.model_validate(await self._send("Page.getFrameTree")) + self._frame_id = frames.frame_tree.frame.id + await self._send( + "Fetch.enable", + { + "patterns": [ + {"urlPattern": "*", "resourceType": "Document", "requestStage": "Request"} + ] + }, + ) + + def _request_paused(self, value: object) -> None: + if self._closing: + return + try: + event = _Paused.model_validate(value) + except ValidationError as error: + self._fail(error) + return + if event.frame_id == self._frame_id: + self._document_pending = True + self._changed.set() + self._spawn(self._paused(event)) + + def _release(self) -> None: + if self._current_permit is not None: + self._current_permit.release() + self._current_permit = None + self._site = None + + async def _paused(self, event: _Paused) -> None: + if event.frame_id == self._frame_id: + async with self._gate: + redirects = ( + self._redirects.pop(event.redirected_request_id, 0) + 1 + if event.redirected_request_id is not None + else 0 + ) + if redirects > MAX_REDIRECTS: + self._fail(FetchError("too_many_redirects", event.request.url)) + return + self._redirects[event.request_id] = redirects + url = event.request.url + if urlsplit(url).scheme in {"http", "https"}: + site = site_key(url) + if site != self._site: + # Release before waiting at the destination so two visits crossing + # between the same sites cannot hold each other's permits. + self._release() + permit = await self._permit(url) + if self._closing: + permit.release() + return + self._current_permit = permit + self._site = site + await self._send("Fetch.continueRequest", {"requestId": event.request_id}) + else: + await self._send("Fetch.continueRequest", {"requestId": event.request_id}) + + def _requested(self, request: PlaywrightRequest) -> None: + if request.is_navigation_request() and request.frame == self._page.main_frame: + self._current_request = request + self._generation += 1 + self._main = None + self._document_pending = True + self._changed.set() + + def _response(self, response: PlaywrightResponse) -> None: + if response.request is self._current_request: + self._main = response + self._changed.set() + + def _request_failed(self, request: PlaywrightRequest) -> None: + if request is not self._current_request: + return + reason = request.failure + if reason == "net::ERR_ABORTED": + return + error = PlaywrightError(reason or "Main navigation failed without a reason") + self._fail(_source_error(error, request.url) or error) + + def _navigated(self, frame: Frame) -> None: + if frame == self._page.main_frame: + # History/hash changes have no new response or DOMContentLoaded event. + if self._main is not None and self._committed_generation != self._generation: + self._committed_generation = self._generation + self._document_pending = False + self._changed.set() + + def _domcontentloaded(self, _: Page) -> None: + if self._committed_generation == self._generation: + self._loaded_generation = self._generation + self._changed.set() + + def _download(self, download: Download) -> None: + self._fail(FetchError("unreachable", download.url)) + self._spawn(download.cancel()) + + def _ready(self) -> bool: + return ( + self._main is not None + and self._committed_generation == self._generation + and self._loaded_generation == self._generation + and not self._document_pending + and not self._tasks + ) + + async def _navigate(self) -> Response: + await self._page.goto(self._url, wait_until="commit", timeout=0) + while True: + self._changed.clear() + response = self._main + if response is not None and response.status not in _CHALLENGES and self._ready(): + generation = self._generation + try: + body = await self._page.content() + except PlaywrightError as error: + if "page is navigating" not in str(error): + raise + continue + if generation == self._generation and response is self._main and self._ready(): + return Response( + self._page.url, + response.status, + "text/html; charset=utf-8", + body.encode("utf-8"), + ) + await self._changed.wait() + + async def run(self) -> Response: + self._connection = await self._context.new_cdp_session(self._page) + self._connection.on("Fetch.requestPaused", self._request_paused) + self._page.on("request", self._requested) + self._page.on("requestfailed", self._request_failed) + self._page.on("response", self._response) + self._page.on("download", self._download) + self._page.on("framenavigated", self._navigated) + self._page.on("domcontentloaded", self._domcontentloaded) + navigation: asyncio.Task[Response] | None = None + try: + await self._configure() + navigation = asyncio.create_task(self._navigate()) + await asyncio.wait((navigation, self._failure), return_when=asyncio.FIRST_COMPLETED) + if self._failure.done(): + raise self._failure.result() + return await navigation + except PlaywrightError as error: + failure = _source_error(error, self._page.url or self._url) + if failure is not None: + raise failure from error + raise + finally: + if navigation is not None and not navigation.done(): + navigation.cancel() + await asyncio.gather(navigation, return_exceptions=True) + + async def close(self) -> None: + self._closing = True + try: + await self._page.close() + finally: + tasks = tuple(self._tasks) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + self._release() diff --git a/crawler/src/crawler/crawl/browser_test.py b/crawler/src/crawler/crawl/browser_test.py new file mode 100644 index 0000000..3298802 --- /dev/null +++ b/crawler/src/crawler/crawl/browser_test.py @@ -0,0 +1,1183 @@ +import asyncio +from collections.abc import Awaitable, Callable + +import httpx +import pytest +from playwright.async_api import Error as PlaywrightError +from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from pydantic import JsonValue + +from crawler.crawl.browser import Browser +from crawler.crawl.fetch import ( + INTERVAL, + MAX_REDIRECTS, + Failure, + Fetcher, + FetchError, + Permit, + Response, +) + + +def run(main: Callable[[], Awaitable[None]]) -> None: + async def bounded() -> None: + async with asyncio.timeout(5): + await main() + + asyncio.run(bounded()) + + +class FakeSession: + def __init__(self, number: int) -> None: + self.number = number + self.closed = False + + async def get(self, url: str) -> Response: + return Response(url, 200, "text/html", f"session {self.number}".encode()) + + async def download(self, url: str, *, page_url: str) -> Response: + return Response(url, 200, "application/pdf", f"session {self.number}: {page_url}".encode()) + + async def close(self) -> None: + self.closed = True + + +class FakeSessions: + def __init__(self) -> None: + self.sessions: list[FakeSession] = [] + + async def session(self) -> FakeSession: + await asyncio.sleep(0) + session = FakeSession(len(self.sessions) + 1) + self.sessions.append(session) + return session + + +class FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def clock(self) -> float: + return self.now + + async def sleep(self, seconds: float) -> None: + self.now += seconds + await asyncio.sleep(0) + + +class FakeAPIResponse: + def __init__(self, response: httpx.Response) -> None: + self.response = response + self.url = str(response.url) + self.status = response.status_code + self.headers = dict(response.headers) + self.disposed = False + + async def body(self) -> bytes: + return self.response.content + + async def dispose(self) -> None: + self.disposed = True + await self.response.aclose() + + +class FakeDownloads: + def __init__(self, client: httpx.AsyncClient) -> None: + self.client = client + self.responses: list[FakeAPIResponse] = [] + self.error: PlaywrightError | None = None + + async def get( + self, + url: str, + *, + headers: dict[str, str], + **options: object, + ) -> FakeAPIResponse: + if self.error is not None: + raise self.error + assert options["max_redirects"] == options["max_retries"] == 0 + assert options["fail_on_status_code"] is False + request_timeout = options["timeout"] + assert isinstance(request_timeout, int | float) + response = FakeAPIResponse( + await self.client.get(url, headers=headers, timeout=request_timeout / 1000) + ) + self.responses.append(response) + return response + + +class FakeCDP: + def __init__(self) -> None: + self.handlers: dict[str, Callable[..., None]] = {} + self.commands: asyncio.Queue[tuple[str, dict[str, JsonValue]]] = asyncio.Queue() + self.continued: dict[str, asyncio.Event] = {} + + def on(self, event: str, callback: Callable[..., None]) -> None: + self.handlers[event] = callback + + async def send( + self, method: str, params: dict[str, JsonValue] | None = None + ) -> dict[str, JsonValue]: + self.commands.put_nowait((method, params or {})) + if method == "Page.getFrameTree": + return {"frameTree": {"frame": {"id": "main"}}} + if method == "Fetch.continueRequest": + assert params is not None + request_id = params["requestId"] + assert isinstance(request_id, str) + self.continued[request_id].set() + return {} + + async def next_command( + self, method: str, *, request_id: str | None = None + ) -> dict[str, JsonValue]: + while True: + name, params = await self.commands.get() + if name == method and (request_id is None or params.get("requestId") == request_id): + return params + + def pause( + self, + request_id: str, + url: str, + *, + frame_id: str = "main", + redirected: str | None = None, + ) -> None: + self.continued[request_id] = asyncio.Event() + value: dict[str, JsonValue] = { + "requestId": request_id, + "request": {"url": url, "method": "GET"}, + "resourceType": "Document", + "frameId": frame_id, + } + if redirected is not None: + value["redirectedRequestId"] = redirected + self.handlers["Fetch.requestPaused"](value) + + +class FakeFrame: + def __init__(self, page: FakePage) -> None: + self.page = page + + +class FakeNavigation: + def __init__(self, frame: FakeFrame, url: str, *, navigation: bool = True) -> None: + self.frame = frame + self.url = url + self.navigation = navigation + self.failure: str | None = None + + def is_navigation_request(self) -> bool: + return self.navigation + + +class FakeDocument: + def __init__(self, status: int, request: FakeNavigation) -> None: + self.request = request + self.status = status + + +class FakePage: + def __init__(self, context: FakeContext) -> None: + self.context = context + self.main_frame = FakeFrame(self) + self.navigation = FakeNavigation(self.main_frame, "about:blank") + self.url = "about:blank" + self.html = "" + self.closed = False + self.close_started = asyncio.Event() + self.close_gate: asyncio.Event | None = None + self.started = asyncio.Event() + self.content_started = asyncio.Event() + self.content_gate: asyncio.Event | None = None + self.navigation_error: PlaywrightError | None = None + self.connection = FakeCDP() + self.handlers: dict[str, Callable[..., None]] = {} + + def on(self, event: str, callback: Callable[..., None]) -> None: + self.handlers[event] = callback + + def begin_navigation(self, url: str) -> FakeNavigation: + self.navigation = FakeNavigation(self.main_frame, url) + self.handlers["request"](self.navigation) + return self.navigation + + def respond(self, status: int, *, request: FakeNavigation | None = None) -> None: + self.handlers["response"](FakeDocument(status, request or self.navigation)) + + def fail(self, request: FakeNavigation, reason: str) -> None: + request.failure = reason + self.handlers["requestfailed"](request) + + def commit(self, url: str, html: str, *, loaded: bool = True) -> None: + self.url = url + self.html = html + if callback := self.handlers.get("framenavigated"): + callback(self.main_frame) + if loaded: + self.load() + + def load(self) -> None: + self.handlers["domcontentloaded"](self) + + async def goto(self, url: str, **_options: object) -> None: + self.begin_navigation(url) + self.connection.pause("initial", url) + await self.connection.continued["initial"].wait() + if self.navigation_error is not None: + raise self.navigation_error + self.respond(self.context.driver.initial_status) + self.commit(url, "challenge") + self.started.set() + + async def content(self) -> str: + html = self.html + self.content_started.set() + if self.content_gate is not None: + await self.content_gate.wait() + return html + + async def close(self) -> None: + if not self.closed: + self.close_started.set() + if self.close_gate is not None: + await self.close_gate.wait() + self.closed = True + self.context.driver.active_pages -= 1 + + +class FakeRouteRequest: + def __init__(self, page: FakePage, resource_type: str) -> None: + self.frame = FakeFrame(page) + self.resource_type = resource_type + + +class FakeRoute: + def __init__(self, page: FakePage, resource_type: str) -> None: + self.request = FakeRouteRequest(page, resource_type) + self.continued = False + self.aborted = False + + async def continue_(self) -> None: + self.continued = True + + async def abort(self) -> None: + self.aborted = True + + +class FakeContext: + def __init__(self, driver: FakeBrowser) -> None: + self.driver = driver + self.client = httpx.AsyncClient(transport=driver.transport) + self.request = FakeDownloads(self.client) + self.pages: list[FakePage] = [] + self.closed = False + self.route_handler: Callable[[FakeRoute], Awaitable[None]] | None = None + self.page_gate: asyncio.Event | None = None + + async def route(self, pattern: str, handler: Callable[[FakeRoute], Awaitable[None]]) -> None: + self.route_handler = handler + + def on(self, event: str, handler: object) -> None: + pass + + async def new_page(self) -> FakePage: + page = FakePage(self) + self.pages.append(page) + self.driver.active_pages += 1 + self.driver.peak_pages = max(self.driver.peak_pages, self.driver.active_pages) + self.driver.created.put_nowait(page) + if self.page_gate is not None: + await self.page_gate.wait() + return page + + async def new_cdp_session(self, page: FakePage) -> FakeCDP: + return page.connection + + async def close(self) -> None: + self.closed = True + for page in self.pages: + await page.close() + await self.client.aclose() + + +class FakeBrowser: + def __init__(self, transport: httpx.AsyncBaseTransport | None = None) -> None: + self.transport = transport or httpx.MockTransport(lambda _: httpx.Response(404)) + self.contexts: list[FakeContext] = [] + self.created: asyncio.Queue[FakePage] = asyncio.Queue() + self.active_pages = 0 + self.peak_pages = 0 + self.initial_status = 202 + self.context_created = asyncio.Event() + self.context_gate: asyncio.Event | None = None + + async def new_context( + self, *, user_agent: str, service_workers: str, accept_downloads: bool + ) -> FakeContext: + context = FakeContext(self) + self.contexts.append(context) + self.context_created.set() + if self.context_gate is not None: + await self.context_gate.wait() + return context + + +def use_browser(monkeypatch: pytest.MonkeyPatch, driver: FakeBrowser) -> None: + async def start(_owner: object) -> FakeBrowser: + return driver + + monkeypatch.setattr("crawler.crawl.browser.Browser._start", start) + + +def test_plain_pages_and_unused_browser_sessions_do_not_start_a_browser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def unexpected_browser(_fetcher: Fetcher) -> None: + pytest.fail("A browser was started without a browser page request") + + monkeypatch.setattr(Fetcher, "browser", unexpected_browser) + + async def main() -> None: + transport = httpx.MockTransport( + lambda request: httpx.Response(200, content=request.url.path) + ) + async with Fetcher(transport) as fetcher: + async with fetcher.session(False) as pages: + listing = await pages.get("https://a/list") + attachment = await pages.download("https://b/file.pdf", page_url=listing.url) + async with fetcher.session(True): + pass + assert listing.body == b"/list" + assert attachment.body == b"/file.pdf" + + run(main) + + +def test_round_reuses_its_browser_session_and_closes_it_before_the_next_round( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = FakeSessions() + + def browser_service(_fetcher: Fetcher) -> FakeSessions: + return service + + monkeypatch.setattr(Fetcher, "browser", browser_service) + + async def main() -> None: + async with Fetcher(httpx.MockTransport(lambda _: httpx.Response(500))) as fetcher: + async with fetcher.session(True) as pages: + assert service.sessions == [] + listing, notice = await asyncio.gather( + pages.get("https://a/list"), pages.get("https://a/notice") + ) + attachment = await pages.download("https://a/file.pdf", page_url=notice.url) + assert listing.body == notice.body == b"session 1" + assert attachment.body == b"session 1: https://a/notice" + assert not service.sessions[0].closed + + with pytest.raises(RuntimeError, match="closed"): + await pages.get("https://a/after-close") + with pytest.raises(RuntimeError, match="closed"): + await pages.download("https://a/file.pdf", page_url=notice.url) + assert service.sessions[0].closed + + async with fetcher.session(True) as next_round: + assert (await next_round.get("https://a/list")).body == b"session 2" + assert service.sessions[0].closed + assert all(session.closed for session in service.sessions) + + run(main) + + +@pytest.mark.parametrize("challenge", [202, 412]) +def test_challenge_waits_for_the_final_document_to_commit_before_reading_html( + monkeypatch: pytest.MonkeyPatch, challenge: int +) -> None: + driver = FakeBrowser() + driver.initial_status = challenge + use_browser(monkeypatch, driver) + + async def main() -> None: + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + await page.started.wait() + await asyncio.sleep(0) + assert not task.done() + + # A response event precedes commit, even when both documents use the same URL. + page.begin_navigation("https://a.test/list") + page.connection.pause("challenge", "https://a.test/list") + await page.connection.next_command("Fetch.continueRequest", request_id="challenge") + page.respond(200) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() + + page.commit("https://a.test/list", "script pending", loaded=False) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() + page.html = "rendered notice" + page.load() + result = await task + assert result.status == 200 + assert result.body == b"rendered notice" + assert page.closed + assert driver.active_pages == 0 + finally: + await browser.close() + + run(main) + + +def test_page_limit_is_shared_across_rounds_and_cancellation_frees_a_slot( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + finish = asyncio.Event() + + async def render(_request: object) -> Response: + await finish.wait() + return Response("https://a.test/list", 200, "text/html", b"rendered") + + monkeypatch.setattr("crawler.crawl.browser._Page.run", render) + + async def main() -> None: + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + try: + first, second = await asyncio.gather(browser.session(), browser.session()) + tasks = [ + asyncio.create_task(first.get("https://a.test/1")), + asyncio.create_task(second.get("https://b.test/2")), + asyncio.create_task(first.get("https://a.test/3")), + ] + page_one = await driver.created.get() + await driver.created.get() + await asyncio.sleep(0) + assert driver.active_pages == driver.peak_pages == 2 + assert driver.created.empty() + + tasks[0].cancel() + with pytest.raises(asyncio.CancelledError): + await tasks[0] + page_three = await driver.created.get() + assert page_one.closed + assert not page_three.closed + assert driver.active_pages == driver.peak_pages == 2 + + finish.set() + assert [response.body for response in await asyncio.gather(*tasks[1:])] == [ + b"rendered", + b"rendered", + ] + assert driver.active_pages == 0 + finally: + finish.set() + await browser.close() + assert all(context.closed for context in driver.contexts) + + run(main) + + +def test_navigation_during_html_read_discards_the_previous_document( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + driver.initial_status = 200 + use_browser(monkeypatch, driver) + + async def main() -> None: + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + page.content_gate = asyncio.Event() + await page.content_started.wait() + previous = page.navigation + page.begin_navigation("https://a.test/final") + page.content_gate.set() + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() + + page.respond(200, request=previous) + await asyncio.sleep(0) + assert not task.done() + page.connection.pause("navigate", "https://a.test/final") + await page.connection.next_command("Fetch.continueRequest", request_id="navigate") + page.respond(200) + await asyncio.sleep(0) + assert not task.done() + page.commit("https://a.test/final", "final document") + result = await task + assert result.url == "https://a.test/final" + assert result.body == b"final document" + finally: + await browser.close() + + run(main) + + +def test_attachment_cookies_are_scoped_to_the_round_and_each_redirect_destination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requested: list[tuple[str, str, str]] = [] + + def handle(request: httpx.Request) -> httpx.Response: + requested.append( + ( + str(request.url), + request.headers.get("cookie", ""), + request.headers.get("referer", ""), + ) + ) + if request.url.host == "a.test": + if request.headers.get("cookie") != "challenge=passed": + return httpx.Response(403, content=b"challenge required") + return httpx.Response(302, headers={"location": "https://b.test/file.pdf"}) + assert "challenge" not in request.headers.get("cookie", "") + return httpx.Response(200, headers={"content-type": "application/pdf"}, content=b"PDF") + + driver = FakeBrowser(httpx.MockTransport(handle)) + use_browser(monkeypatch, driver) + clock = FakeClock() + + async def main() -> None: + async with Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + context = driver.contexts[0] + context.client.cookies.set("challenge", "passed", domain="a.test", path="/private") + result = await session.download( + "https://a.test/private/file.pdf", page_url="https://a.test/notice" + ) + assert (result.url, result.status, result.body) == ( + "https://b.test/file.pdf", + 200, + b"PDF", + ) + assert all(response.disposed for response in context.request.responses) + outside_cookie_path = await session.download( + "https://a.test/public/file.pdf", page_url="https://a.test/notice" + ) + assert outside_cookie_path.status == 403 + await session.close() + + next_round = await browser.session() + missing_cookie = await next_round.download( + "https://a.test/private/file.pdf", page_url="https://a.test/notice" + ) + assert missing_cookie.status == 403 + assert clock.now >= INTERVAL + finally: + await browser.close() + + assert requested == [ + ("https://a.test/private/file.pdf", "challenge=passed", "https://a.test/notice"), + ("https://b.test/file.pdf", "", "https://a.test/notice"), + ("https://a.test/public/file.pdf", "", "https://a.test/notice"), + ("https://a.test/private/file.pdf", "", "https://a.test/notice"), + ] + + run(main) + + +def test_page_permit_covers_challenge_refresh_and_subresources_until_the_page_closes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + clock = FakeClock() + ordinary_started = asyncio.Event() + permits: list[str] = [] + + def handle(_request: httpx.Request) -> httpx.Response: + ordinary_started.set() + return httpx.Response(200, content=b"next request") + + async def main() -> None: + async with Fetcher(httpx.MockTransport(handle), clock.clock, clock.sleep) as fetcher: + + async def permit(url: str) -> Permit: + permits.append(url) + return await fetcher.permit(url) + + browser = Browser(permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + await page.started.wait() + configuration = await page.connection.next_command("Fetch.enable") + assert configuration == { + "patterns": [ + {"urlPattern": "*", "requestStage": "Request", "resourceType": "Document"} + ] + } + + route = driver.contexts[0].route_handler + assert route is not None + for resource_type in ("script", "stylesheet", "xhr", "document"): + resource = FakeRoute(page, resource_type) + await route(resource) + assert resource.continued + assert not resource.aborted + image = FakeRoute(page, "image") + await route(image) + assert image.aborted + + page.connection.pause("iframe", "https://b.test/frame", frame_id="child") + await page.connection.next_command("Fetch.continueRequest", request_id="iframe") + page.begin_navigation("https://a.test/list") + page.connection.pause("challenge", "https://a.test/list") + await page.connection.next_command("Fetch.continueRequest", request_id="challenge") + assert permits == ["https://a.test/list"] + assert clock.now == 0 + + ordinary = asyncio.create_task(fetcher.get("https://a.test/other")) + await asyncio.sleep(0) + assert not ordinary_started.is_set() + page.respond(200) + page.commit("https://a.test/list", "ready") + assert (await task).body == b"ready" + assert page.closed + assert (await ordinary).body == b"next request" + assert clock.now == INTERVAL + finally: + await browser.close() + + run(main) + + +def test_opposite_cross_site_redirects_release_old_permits_before_waiting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + clock = FakeClock() + + async def main() -> None: + async with Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + first = asyncio.create_task(session.get("https://a.test/list")) + second = asyncio.create_task(session.get("https://b.test/list")) + page_a = await driver.created.get() + page_b = await driver.created.get() + await asyncio.gather(page_a.started.wait(), page_b.started.wait()) + page_a.begin_navigation("https://b.test/final") + page_b.begin_navigation("https://a.test/final") + page_a.connection.pause("to-b", "https://b.test/final", redirected="initial") + page_b.connection.pause("to-a", "https://a.test/final", redirected="initial") + await asyncio.gather( + page_a.connection.next_command("Fetch.continueRequest", request_id="to-b"), + page_b.connection.next_command("Fetch.continueRequest", request_id="to-a"), + ) + + page_a.respond(200) + page_a.commit("https://b.test/final", "from b") + page_b.respond(200) + page_b.commit("https://a.test/final", "from a") + responses = await asyncio.gather(first, second) + assert [(response.url, response.body) for response in responses] == [ + ("https://b.test/final", b"from b"), + ("https://a.test/final", b"from a"), + ] + assert page_a.closed + assert page_b.closed + assert clock.now >= INTERVAL + finally: + await browser.close() + + run(main) + + +def test_same_document_url_change_during_html_read_keeps_document_readiness( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + driver.initial_status = 200 + use_browser(monkeypatch, driver) + + async def main() -> None: + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + page.content_gate = asyncio.Event() + await page.content_started.wait() + page.commit("https://a.test/list?view=all#notices", page.html, loaded=False) + page.content_gate.set() + result = await task + assert result.url == "https://a.test/list?view=all#notices" + assert result.body == b"challenge" + assert page.closed + finally: + await browser.close() + + run(main) + + +@pytest.mark.parametrize( + ("error", "failure"), + [ + (None, None), + (PlaywrightTimeoutError("Page.goto timed out"), "timeout"), + (PlaywrightError("net::ERR_CONNECTION_RESET"), "unreachable"), + (PlaywrightError("Page.goto: Download is starting"), "unreachable"), + (PlaywrightError("Unexpected browser error"), None), + ], + ids=["cancel", "timeout", "unreachable", "unexpected_download", "unexpected_error"], +) +def test_navigation_failure_or_cancellation_closes_the_page_and_releases_its_permit( + monkeypatch: pytest.MonkeyPatch, + error: PlaywrightError | None, + failure: Failure | None, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + clock = FakeClock() + + async def main() -> None: + transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"still usable")) + async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + if error is None: + await page.started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + page.navigation_error = error + if failure is None: + with pytest.raises(PlaywrightError) as browser_error: + await task + assert browser_error.value is error + else: + with pytest.raises(FetchError) as source_failure: + await task + assert source_failure.value.failure == failure + assert page.closed + assert driver.active_pages == 0 + assert (await fetcher.get("https://a.test/after-failure")).body == b"still usable" + assert clock.now == INTERVAL + finally: + await browser.close() + + run(main) + + +def test_cancelling_a_cross_site_wait_leaves_both_sites_usable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + clock = FakeClock() + destination_requested = asyncio.Event() + + async def main() -> None: + transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"available")) + async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: + + async def permit(url: str) -> Permit: + if url == "https://b.test/final": + destination_requested.set() + return await fetcher.permit(url) + + browser = Browser(permit) + destination = await fetcher.permit("https://b.test/held") + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + await page.started.wait() + page.begin_navigation("https://b.test/final") + page.connection.pause("redirect", "https://b.test/final", redirected="initial") + await destination_requested.wait() + assert (await fetcher.get("https://a.test/other")).body == b"available" + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert page.closed + destination.release() + assert (await fetcher.get("https://b.test/other")).body == b"available" + finally: + destination.release() + await browser.close() + + run(main) + + +def test_same_site_http_redirect_limit_does_not_count_challenge_refreshes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + clock = FakeClock() + + async def main() -> None: + transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"available")) + async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + await page.started.wait() + for index in range(MAX_REDIRECTS + 2): + request_id = f"challenge-{index}" + page.begin_navigation("https://a.test/list") + page.connection.pause(request_id, "https://a.test/list") + await page.connection.next_command( + "Fetch.continueRequest", request_id=request_id + ) + previous = "initial" + for index in range(MAX_REDIRECTS): + request_id = f"redirect-{index}" + page.begin_navigation(f"https://a.test/redirect/{index}") + page.connection.pause( + request_id, f"https://a.test/redirect/{index}", redirected=previous + ) + await page.connection.next_command( + "Fetch.continueRequest", request_id=request_id + ) + previous = request_id + page.begin_navigation("https://a.test/too-far") + page.connection.pause("over-limit", "https://a.test/too-far", redirected=previous) + with pytest.raises(FetchError) as caught: + await task + assert caught.value.failure == "too_many_redirects" + assert not page.connection.continued["over-limit"].is_set() + assert page.closed + assert (await fetcher.get("https://a.test/other")).body == b"available" + assert clock.now == INTERVAL + finally: + await browser.close() + + run(main) + + +def test_challenge_navigation_network_failure_releases_the_page_without_waiting_for_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + clock = FakeClock() + + async def main() -> None: + transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"available")) + async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + await page.started.wait() + page.begin_navigation("https://a.test/after-challenge") + page.connection.pause("challenge", "https://a.test/after-challenge") + await page.connection.next_command("Fetch.continueRequest", request_id="challenge") + page.fail(page.navigation, "net::ERR_CONNECTION_REFUSED") + + with pytest.raises(FetchError) as caught: + await task + assert caught.value.failure == "unreachable" + assert caught.value.url == "https://a.test/after-challenge" + assert page.closed + assert (await fetcher.get("https://a.test/other")).body == b"available" + assert clock.now == INTERVAL + finally: + await browser.close() + + run(main) + + +def test_aborted_navigation_and_old_or_subresource_failures_allow_the_replacement_document( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + + async def main() -> None: + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + await page.started.wait() + previous = page.navigation + page.fail(previous, "net::ERR_ABORTED") + await asyncio.sleep(0) + assert not task.done() + + page.begin_navigation("https://a.test/final") + page.fail(previous, "net::ERR_CONNECTION_REFUSED") + script = FakeNavigation( + page.main_frame, "https://a.test/optional.js", navigation=False + ) + page.fail(script, "net::ERR_CONNECTION_REFUSED") + iframe = FakeNavigation(FakeFrame(page), "https://b.test/optional-frame") + page.fail(iframe, "net::ERR_CONNECTION_REFUSED") + await asyncio.sleep(0) + assert not task.done() + + page.connection.pause("replacement", "https://a.test/final") + await page.connection.next_command( + "Fetch.continueRequest", request_id="replacement" + ) + page.respond(200) + page.commit("https://a.test/final", "replacement document") + assert (await task).body == b"replacement document" + assert page.closed + finally: + await browser.close() + + run(main) + + +def test_cancelling_context_creation_waits_for_and_closes_the_unclaimed_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + driver = FakeBrowser() + driver.context_gate = asyncio.Event() + use_browser(monkeypatch, driver) + + async def main() -> None: + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + task = asyncio.create_task(browser.session()) + try: + await driver.context_created.wait() + task.cancel() + await asyncio.sleep(0) + assert not task.done() + assert not driver.contexts[0].closed + assert driver.context_gate is not None + driver.context_gate.set() + with pytest.raises(asyncio.CancelledError): + await task + assert driver.contexts[0].closed + assert not browser.sessions + finally: + assert driver.context_gate is not None + driver.context_gate.set() + await asyncio.gather(task, return_exceptions=True) + await browser.close() + for context in driver.contexts: + await context.close() + + run(main) + + +@pytest.mark.parametrize("expired", [False, True], ids=["cancel", "timeout"]) +def test_interrupted_page_creation_closes_the_page_before_releasing_its_slot( + monkeypatch: pytest.MonkeyPatch, expired: bool +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + + async def main() -> None: + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + session = await browser.session() + context = driver.contexts[0] + context.page_gate = asyncio.Event() + deadlines: list[asyncio.Timeout] = [] + timeout = asyncio.timeout + + def deadline(_seconds: float) -> asyncio.Timeout: + timer = timeout(None) + deadlines.append(timer) + return timer + + monkeypatch.setattr("crawler.crawl.browser.asyncio.timeout", deadline) + task = asyncio.create_task(session.get("https://a.test/list")) + page = await driver.created.get() + page.close_gate = asyncio.Event() + blocked: asyncio.Task[bool] | None = None + try: + if expired: + deadlines[0].reschedule(asyncio.get_running_loop().time()) + else: + task.cancel() + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not task.done() + context.page_gate.set() + await page.close_started.wait() + await browser.slots.acquire() + blocked = asyncio.create_task(browser.slots.acquire()) + await asyncio.sleep(0) + assert not blocked.done() + assert not page.closed + page.close_gate.set() + if expired: + with pytest.raises(FetchError) as caught: + await task + assert caught.value.failure == "timeout" + else: + with pytest.raises(asyncio.CancelledError): + await task + await blocked + browser.slots.release() + browser.slots.release() + assert page.closed + assert driver.active_pages == 0 + finally: + context.page_gate.set() + page.close_gate.set() + if blocked is not None and not blocked.done(): + blocked.cancel() + await asyncio.gather(task, return_exceptions=True) + if blocked is not None: + await asyncio.gather(blocked, return_exceptions=True) + await browser.close() + + run(main) + + +@pytest.mark.parametrize("page_creation", [False, True], ids=["context", "page"]) +def test_creation_failure_after_cancellation_is_propagated_once( + monkeypatch: pytest.MonkeyPatch, page_creation: bool +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + + async def main() -> None: + started, finish = asyncio.Event(), asyncio.Event() + error = RuntimeError("Resource creation failed") + unhandled: list[dict[str, object]] = [] + loop = asyncio.get_running_loop() + previous_handler = loop.get_exception_handler() + + def unexpected(_loop: asyncio.AbstractEventLoop, context: dict[str, object]) -> None: + unhandled.append(context) + + loop.set_exception_handler(unexpected) + async with Fetcher() as fetcher: + browser = Browser(fetcher.permit) + if page_creation: + session = await browser.session() + + async def create_page() -> FakePage: + started.set() + await finish.wait() + raise error + + monkeypatch.setattr(driver.contexts[0], "new_page", create_page) + + async def operation() -> object: + return await session.get("https://a.test/list") + + else: + + async def create_context(**_options: object) -> FakeContext: + started.set() + await finish.wait() + raise error + + monkeypatch.setattr(driver, "new_context", create_context) + + async def operation() -> object: + return await browser.session() + + task = asyncio.create_task(operation()) + try: + await started.wait() + task.cancel() + await asyncio.sleep(0) + assert not task.done() + finish.set() + with pytest.raises(RuntimeError) as caught: + await task + assert caught.value is error + await asyncio.sleep(0) + assert not unhandled + assert driver.active_pages == 0 + finally: + finish.set() + await asyncio.gather(task, return_exceptions=True) + loop.set_exception_handler(previous_handler) + await browser.close() + + run(main) + + +@pytest.mark.parametrize( + ("message", "failure"), + [ + ("APIRequestContext.get: self-signed certificate", "unreachable"), + ("APIRequestContext.get: certificate has expired", "unreachable"), + ("APIRequestContext.get: unable to verify the first certificate", "unreachable"), + ( + "APIRequestContext.get: Hostname/IP does not match certificate's altnames:" + " Host: a.test", + "unreachable", + ), + ( + "APIRequestContext.get: failed to decompress 'gzip' encoding: incorrect header", + "unreachable", + ), + ("APIRequestContext.get: Unknown internal failure", None), + ( + "APIRequestContext.get: Unknown internal failure\nCall log:\n" + " - User-Agent: ECONNRESET self-signed certificate\n", + None, + ), + ], + ids=["self_signed", "expired", "unverified", "hostname", "compression", "unknown", "call_log"], +) +def test_attachment_transport_errors_release_the_permit_and_preserve_unknown_errors( + monkeypatch: pytest.MonkeyPatch, message: str, failure: Failure | None +) -> None: + driver = FakeBrowser() + use_browser(monkeypatch, driver) + clock = FakeClock() + + async def main() -> None: + transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"still available")) + async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: + browser = Browser(fetcher.permit) + try: + session = await browser.session() + error = PlaywrightError(message) + driver.contexts[0].request.error = error + if failure is None: + with pytest.raises(PlaywrightError) as caught: + await session.download( + "https://a.test/file.pdf", page_url="https://a.test/notice" + ) + assert caught.value is error + else: + with pytest.raises(FetchError) as source_failure: + await session.download( + "https://a.test/file.pdf", page_url="https://a.test/notice" + ) + assert source_failure.value.failure == failure + assert ( + await fetcher.get("https://a.test/after-failure") + ).body == b"still available" + assert clock.now == INTERVAL + finally: + await browser.close() + + run(main) diff --git a/crawler/src/crawler/crawl/contract.py b/crawler/src/crawler/crawl/contract.py index 925d454..e3cce86 100644 --- a/crawler/src/crawler/crawl/contract.py +++ b/crawler/src/crawler/crawl/contract.py @@ -29,7 +29,7 @@ class PendingNotice(BaseModel): step: Annotated[ Literal["screen", "fetch"], Field( - description="screen：还没初筛；fetch：要抓正文，初筛通过还没抽取的，或者项目还没截止、一天没抓过的" + description="screen：还没初筛；fetch：初筛通过但最新快照还没抽取的，或者项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的" ), ] @@ -114,6 +114,14 @@ class AttachmentInfo(BaseModel): url: Annotated[str, Field(max_length=2000)] content_type: Annotated[str, Field(max_length=200, min_length=1)] text: Annotated[str, Field(max_length=2097152)] + error: Annotated[ + str | None, + Field( + description="附件下载失败原因；不写表示下载正常，不影响通知正文和其他附件", + max_length=2000, + min_length=1, + ), + ] = None type ValueItem = Annotated[str, Field(max_length=500, min_length=1)] diff --git a/crawler/src/crawler/crawl/fetch.py b/crawler/src/crawler/crawl/fetch.py index 35a43b4..9897916 100644 --- a/crawler/src/crawler/crawl/fetch.py +++ b/crawler/src/crawler/crawl/fetch.py @@ -1,18 +1,19 @@ -"""Every request to a source site goes through here, so the crawl keeps its manners in one place -(design §7.3): it says who it is and sends one request at a time to a site with a pause between -them.""" +"""Source access shares one policy here (design §7.3): identify the crawler and allow one +page visit or HTTP request at a time per site, with a pause between them.""" import asyncio import time from collections.abc import Awaitable, Callable from dataclasses import dataclass, field -from typing import Literal, Self -from urllib.parse import urlsplit +from typing import TYPE_CHECKING, Literal, Self import httpx +if TYPE_CHECKING: + from crawler.crawl.browser import Browser, BrowserSession + USER_AGENT = "mouseclock/0.1 (+https://github.com/mousebaoyan/mouseclock)" -# Seconds from the end of one request to a site to the start of the next. +# Seconds from the end of one page visit or HTTP request to the start of the next. INTERVAL = 5.0 MAX_REDIRECTS = 5 TIMEOUT = 30.0 @@ -41,6 +42,23 @@ class _Site: finished_at: float = float("-inf") +def site_key(url: str) -> str: + return httpx.URL(url).netloc.decode("ascii") + + +@dataclass +class Permit: + _site: _Site + _clock: Callable[[], float] + _released: bool = False + + def release(self) -> None: + if not self._released: + self._released = True + self._site.finished_at = self._clock() + self._site.lock.release() + + class Fetcher: """One per crawler process, shared by everything that crawls, so the rules hold across tasks.""" @@ -56,12 +74,40 @@ def __init__( self._clock = clock self._sleep = sleep self._sites: dict[str, _Site] = {} + self._browser: Browser | None = None async def __aenter__(self) -> Self: return self async def __aexit__(self, *_: object) -> None: - await self._client.aclose() + try: + if self._browser is not None: + await self._browser.close() + finally: + await self._client.aclose() + + def session(self, browser: bool) -> PageSession: + return PageSession(self, browser) + + def browser(self) -> Browser: + if self._browser is None: + # Keep the browser library unloaded until a source actually needs it. + from crawler.crawl.browser import Browser + + self._browser = Browser(self.permit) + return self._browser + + async def permit(self, url: str) -> Permit: + site = self._sites.setdefault(site_key(url), _Site()) + await site.lock.acquire() + try: + wait = site.finished_at + INTERVAL - self._clock() + if wait > 0: + await self._sleep(wait) + return Permit(site, self._clock) + except BaseException: + site.lock.release() + raise async def get(self, url: str) -> Response: # Each hop on its own, so a redirect to another site waits its turn there too. @@ -78,16 +124,49 @@ async def get(self, url: str) -> Response: raise FetchError("too_many_redirects", url) async def _send(self, url: str) -> httpx.Response: - site = self._sites.setdefault(urlsplit(url).netloc, _Site()) - async with site.lock: - wait = site.finished_at + INTERVAL - self._clock() - if wait > 0: - await self._sleep(wait) - try: - return await self._client.get(url) - except httpx.TimeoutException as error: - raise FetchError("timeout", url) from error - except httpx.TransportError as error: - raise FetchError("unreachable", url) from error - finally: - site.finished_at = self._clock() + permit = await self.permit(url) + try: + return await self._client.get(url) + except httpx.TimeoutException as error: + raise FetchError("timeout", url) from error + except httpx.TransportError as error: + raise FetchError("unreachable", url) from error + finally: + permit.release() + + +class PageSession: + def __init__(self, fetcher: Fetcher, browser: bool) -> None: + self._fetcher = fetcher + self._use_browser = browser + self._browser_session: BrowserSession | None = None + self._lock = asyncio.Lock() + self._closed = False + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *_: object) -> None: + self._closed = True + if self._browser_session is not None: + await self._browser_session.close() + + async def _open_browser(self) -> BrowserSession: + async with self._lock: + if self._browser_session is None: + self._browser_session = await self._fetcher.browser().session() + return self._browser_session + + async def get(self, url: str) -> Response: + if self._closed: + raise RuntimeError("Page session is closed") + if self._use_browser: + return await (await self._open_browser()).get(url) + return await self._fetcher.get(url) + + async def download(self, url: str, *, page_url: str) -> Response: + if self._closed: + raise RuntimeError("Page session is closed") + if self._use_browser: + return await (await self._open_browser()).download(url, page_url=page_url) + return await self._fetcher.get(url) diff --git a/crawler/src/crawler/crawl/fetch_test.py b/crawler/src/crawler/crawl/fetch_test.py index 01de10d..4b2f516 100644 --- a/crawler/src/crawler/crawl/fetch_test.py +++ b/crawler/src/crawler/crawl/fetch_test.py @@ -5,8 +5,9 @@ from urllib.parse import urlsplit import httpx +import pytest -from crawler.crawl.fetch import INTERVAL, USER_AGENT, Fetcher +from crawler.crawl.fetch import INTERVAL, USER_AGENT, Fetcher, FetchError class FakeSites: @@ -85,3 +86,90 @@ def test_a_redirect_to_another_site_waits_its_turn_there() -> None: assert moved.url == "http://b/new" assert len(sites.times("b")) == 2 assert all(gap >= INTERVAL for gap in gaps(sites.times("b"))) + + +def test_http_and_browser_permissions_share_the_completion_interval() -> None: + sites = FakeSites({"http://a/notice": httpx.Response(200)}) + + async def use(fetcher: Fetcher) -> None: + browser_request = await fetcher.permit("http://A:80/list.htm") + http_request = asyncio.create_task(fetcher.get("http://a/notice")) + await asyncio.sleep(0) + assert not sites.requests + sites.now = 3.0 + browser_request.release() + # A late terminal event for the old request must not release another one's permit. + browser_request.release() + await http_request + + sites.run(use) + assert sites.times("a") == [3.0 + INTERVAL] + + +def test_cancelling_the_interval_wait_does_not_mark_a_request_finished() -> None: + async def run() -> None: + sites = FakeSites({}) + waiting = asyncio.Event() + waits: list[float] = [] + block = True + + async def sleep(seconds: float) -> None: + waits.append(seconds) + if block: + waiting.set() + await asyncio.Future[None]() + sites.now += seconds + + async with Fetcher(clock=sites.clock, sleep=sleep) as fetcher: + first = await fetcher.permit("http://a/first") + first.release() + queued = asyncio.create_task(fetcher.permit("http://a/cancelled")) + await waiting.wait() + queued.cancel() + with pytest.raises(asyncio.CancelledError): + await queued + block = False + async with asyncio.timeout(1): + next_request = await fetcher.permit("http://a/next") + next_request.release() + assert waits == [INTERVAL, INTERVAL] + + asyncio.run(run()) + + +@pytest.mark.parametrize("failure", ["timeout", "unreachable", "cancelled"]) +def test_failed_or_cancelled_http_requests_release_the_shared_permission(failure: str) -> None: + async def run() -> None: + sites = FakeSites({}) + started = asyncio.Event() + requests = 0 + + async def handle(request: httpx.Request) -> httpx.Response: + nonlocal requests + requests += 1 + if requests == 1: + started.set() + if failure == "cancelled": + await asyncio.Future[None]() + if failure == "timeout": + raise httpx.ReadTimeout("Timed out", request=request) + raise httpx.ConnectError("Connection failed", request=request) + return httpx.Response(200) + + async with Fetcher(httpx.MockTransport(handle), sites.clock, sites.sleep) as fetcher: + task = asyncio.create_task(fetcher.get("http://a/first")) + await started.wait() + if failure == "cancelled": + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + with pytest.raises(FetchError) as caught: + await task + assert caught.value.failure == failure + async with asyncio.timeout(1): + response = await fetcher.get("http://a/next") + assert response.status == 200 + assert sites.now == INTERVAL + + asyncio.run(run()) diff --git a/crawler/src/crawler/crawl/rounds.py b/crawler/src/crawler/crawl/rounds.py index 78d87d2..beef517 100644 --- a/crawler/src/crawler/crawl/rounds.py +++ b/crawler/src/crawler/crawl/rounds.py @@ -7,8 +7,8 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass +from crawler.crawl.attachments import Attachment from crawler.crawl.contract import ( - AttachmentInfo, ExtractedProgram, Extraction, ExtractionContext, @@ -24,7 +24,7 @@ Usage, WorkSource, ) -from crawler.crawl.fetch import Fetcher, FetchError +from crawler.crawl.fetch import Fetcher, FetchError, PageSession from crawler.crawl.website import Website logger = logging.getLogger(__name__) @@ -39,12 +39,6 @@ def __init__(self, failure: Failure, detail: str | None = None) -> None: self.detail = detail -@dataclass(frozen=True) -class Attachment: - info: AttachmentInfo - data: bytes - - @dataclass(frozen=True) class NoticePage: html: str @@ -57,14 +51,14 @@ class NoticePage: @dataclass(frozen=True) class Steps: # The items on the source's list page, paging back two years when it is new (design §7.1). - list_items: Callable[[Fetcher, WorkSource], Awaitable[list[ListedItem]]] + list_items: Callable[[PageSession, WorkSource], Awaitable[list[ListedItem]]] # A score for each notice by its title alone (design §8.1). screen: Callable[ [WorkSource, list[PendingNotice]], Awaitable[tuple[list[ScreeningResult], list[Usage]]] ] # A notice's page with its attachments downloaded and read; None when the page answers # 404 or 410 (design §7.4). - read: Callable[[Fetcher, WorkSource, str], Awaitable[NoticePage | None]] + read: Callable[[PageSession, WorkSource, str], Awaitable[NoticePage | None]] # The programs a notice is about (design §8.2). extract: Callable[ [NoticePage, ExtractionContext], Awaitable[tuple[list[ExtractedProgram], list[Usage]]] @@ -84,62 +78,63 @@ async def crawl_source( steps: Steps, clock: Callable[[], float] = time.time, ) -> None: - started_at = int(clock()) - try: - items = await steps.list_items(fetcher, source) - if not items: - raise StepError("no_items") - except (FetchError, StepError) as error: - logger.warning("Listing failed: %s", error, extra={"source_id": source.id}) - failure, detail = _failure(error) + async with fetcher.session(source.browser) as pages: + started_at = int(clock()) + try: + items = await steps.list_items(pages, source) + if not items: + raise StepError("no_items") + except (FetchError, StepError) as error: + logger.warning("Listing failed: %s", error, extra={"source_id": source.id}) + failure, detail = _failure(error) + await website.run( + Run( + source_id=source.id, + started_at=started_at, + finished_at=int(clock()), + failure=failure, + detail=detail, + listed=0, + new_count=0, + useful_count=0, + notice_failures=[], + ) + ) + return + + listing = await website.listing(source.id, Listing(items=items)) + new = [notice for notice in listing.pending if notice.step == "screen"] + to_fetch = [notice for notice in listing.pending if notice.step == "fetch"] + if new: + results, usage = await steps.screen(source, new) + await website.screening(source.id, Screening(results=results, usage=usage)) + screened_in = {result.notice_id for result in results if result.screened_in} + to_fetch += [notice for notice in new if notice.id in screened_in] + + useful = 0 + failures: list[NoticeFailure] = [] + for notice in to_fetch: + try: + programs = await _fetch_notice(notice, source, pages, website, steps, clock) + except (FetchError, StepError) as error: + logger.warning("Fetching a notice failed: %s", error, extra={"url": notice.url}) + failure, detail = _failure(error) + failures.append(NoticeFailure(url=notice.url, failure=failure, detail=detail)) + continue + if programs and notice.step == "screen": + useful += 1 + await website.run( Run( source_id=source.id, started_at=started_at, finished_at=int(clock()), - failure=failure, - detail=detail, - listed=0, - new_count=0, - useful_count=0, - notice_failures=[], + listed=len(items), + new_count=listing.new_count, + useful_count=useful, + notice_failures=failures, ) ) - return - - listing = await website.listing(source.id, Listing(items=items)) - new = [notice for notice in listing.pending if notice.step == "screen"] - to_fetch = [notice for notice in listing.pending if notice.step == "fetch"] - if new: - results, usage = await steps.screen(source, new) - await website.screening(source.id, Screening(results=results, usage=usage)) - screened_in = {result.notice_id for result in results if result.screened_in} - to_fetch += [notice for notice in new if notice.id in screened_in] - - useful = 0 - failures: list[NoticeFailure] = [] - for notice in to_fetch: - try: - programs = await _fetch_notice(notice, source, fetcher, website, steps, clock) - except (FetchError, StepError) as error: - logger.warning("Fetching a notice failed: %s", error, extra={"url": notice.url}) - failure, detail = _failure(error) - failures.append(NoticeFailure(url=notice.url, failure=failure, detail=detail)) - continue - if programs and notice.step == "screen": - useful += 1 - - await website.run( - Run( - source_id=source.id, - started_at=started_at, - finished_at=int(clock()), - listed=len(items), - new_count=listing.new_count, - useful_count=useful, - notice_failures=failures, - ) - ) # Returns how many programs the notice turned out to be about; none when it was gone, @@ -147,12 +142,12 @@ async def crawl_source( async def _fetch_notice( notice: PendingNotice, source: WorkSource, - fetcher: Fetcher, + pages: PageSession, website: Website, steps: Steps, clock: Callable[[], float], ) -> int: - page = await steps.read(fetcher, source, notice.url) + page = await steps.read(pages, source, notice.url) if page is None: await website.removed(notice.id) return 0 @@ -166,7 +161,11 @@ async def _fetch_notice( ), ) for position in result.missing_files: - await website.attachment(result.snapshot_id, position, page.attachments[position].data) + attachment = page.attachments[position] + if attachment.data is not None: + await website.attachment( + result.snapshot_id, position, attachment.data, url=attachment.info.url + ) if result.extraction is None: return 0 programs, usage = await steps.extract(page, result.extraction) diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index 0b06365..620976b 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -1,10 +1,14 @@ import asyncio import json import re +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager from typing import Any import httpx +import pytest +from crawler.crawl.attachments import Attachment, download_attachment from crawler.crawl.contract import ( AttachmentInfo, ExtractedFields, @@ -16,8 +20,8 @@ Usage, WorkSource, ) -from crawler.crawl.fetch import Fetcher, FetchError -from crawler.crawl.rounds import Attachment, NoticePage, StepError, Steps, crawl_source +from crawler.crawl.fetch import Fetcher, FetchError, PageSession, Response +from crawler.crawl.rounds import NoticePage, StepError, Steps, crawl_source from crawler.crawl.website import Website SOURCE = WorkSource( @@ -41,11 +45,14 @@ def __init__(self, pending: list[dict[str, Any]], extracted: frozenset[int]) -> self.pending = pending self.extracted = extracted self.calls: list[tuple[str, Any]] = [] + self.upload_urls: list[str] = [] def handle(self, request: httpx.Request) -> httpx.Response: path = request.url.path.removeprefix("/api/v1/crawler") sent_json = request.headers.get("content-type") == "application/json" body: Any = json.loads(request.content) if sent_json else request.content + if request.method == "PUT": + self.upload_urls.append(request.url.params["url"]) self.calls.append((f"{request.method} {path}", body)) if path.endswith("/listing"): return httpx.Response(200, json={"new_count": 2, "pending": self.pending}) @@ -63,14 +70,19 @@ def handle(self, request: httpx.Request) -> httpx.Response: ) return httpx.Response(204) - def run(self, steps: Steps) -> None: + def run( + self, + steps: Steps, + source: WorkSource = SOURCE, + transport: httpx.AsyncBaseTransport | None = None, + ) -> None: async def main() -> None: - nowhere = httpx.MockTransport(lambda _: httpx.Response(500)) + nowhere = transport or httpx.MockTransport(lambda _: httpx.Response(500)) async with ( Fetcher(nowhere) as fetcher, Website("http://site", "token", httpx.MockTransport(self.handle)) as website, ): - await crawl_source(SOURCE, fetcher, website, steps, clock=lambda: 1_800_000_000) + await crawl_source(source, fetcher, website, steps, clock=lambda: 1_800_000_000) asyncio.run(main()) @@ -78,7 +90,7 @@ def called(self, prefix: str) -> list[Any]: return [body for call, body in self.calls if call.startswith(prefix)] -async def list_items(_fetcher: Fetcher, _source: WorkSource) -> list[ListedItem]: +async def list_items(_pages: PageSession, _source: WorkSource) -> list[ListedItem]: return ITEMS @@ -124,7 +136,7 @@ def test_a_round_screens_the_new_fetches_what_needs_it_and_reports() -> None: extracted=frozenset({3}), ) - async def read(_fetcher: Fetcher, _source: WorkSource, url: str) -> NoticePage | None: + async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None: return {"http://a/1.htm": page("冬令营", files=1), "http://a/3.htm": page("夏令营")}.get( url ) @@ -164,33 +176,80 @@ async def read(_fetcher: Fetcher, _source: WorkSource, url: str) -> NoticePage | } +def test_attachment_errors_are_reported_without_uploading_error_pages_or_losing_the_notice() -> ( + None +): + site = FakeWebsite([pending(1, "fetch")], extracted=frozenset()) + files = { + "https://captcha.test/download": "
请输入验证码下载附件
".encode(), + "https://deleted.test/download": "
获取文件错误
".encode(), + "https://files.test/download?file=form.pdf&owner=2": b"%PDF-1.7\n", + } + + def handle(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + content=files[str(request.url)], + headers={"content-type": "text/html;charset=utf-8"}, + ) + + async def read(pages: PageSession, _source: WorkSource, url: str) -> NoticePage: + attachments = [ + await download_attachment(pages, link, name=f"申请表 {n}", page_url=url) + for n, link in enumerate(files) + ] + return NoticePage("

通知

", "通知", attachments) + + site.run(Steps(list_items, screen, read, extract), transport=httpx.MockTransport(handle)) + + [snapshot] = site.called("POST /notices/1/snapshot") + assert [a["url"] for a in snapshot["attachments"]] == list(files) + assert [a.get("error") for a in snapshot["attachments"]] == [ + "下载附件需要验证码", + "获取文件错误", + None, + ] + assert [a["text"] for a in snapshot["attachments"]] == ["", "", ""] + uploads = [(call, body) for call, body in site.calls if call.startswith("PUT ")] + assert uploads == [("PUT /snapshots/10/attachments/2", b"%PDF-1.7\n")] + assert site.upload_urls == ["https://files.test/download?file=form.pdf&owner=2"] + assert len(site.called("POST /snapshots/10/extraction")) == 1 + [run] = site.called("POST /runs") + assert run["notice_failures"] == [] + + def test_notices_that_fail_are_reported_and_the_round_goes_on() -> None: - site = FakeWebsite( - [pending(1, "fetch"), pending(2, "fetch"), pending(3, "fetch")], extracted=frozenset() - ) + site = FakeWebsite([pending(n, "fetch") for n in range(1, 5)], extracted=frozenset()) - async def read(_fetcher: Fetcher, _source: WorkSource, url: str) -> NoticePage | None: + async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None: if url == "http://a/1.htm": raise FetchError("timeout", url) if url == "http://a/2.htm": raise StepError("http_status", "HTTP 403") + if url == "http://a/3.htm": + raise FetchError("unreachable", "http://a/files/application.pdf") return page("冬令营") site.run(Steps(list_items, screen, read, extract)) - assert len(site.called("POST /snapshots/30/extraction")) == 1 + assert len(site.called("POST /snapshots/40/extraction")) == 1 [run] = site.called("POST /runs") assert run["notice_failures"] == [ {"url": "http://a/1.htm", "failure": "timeout", "detail": "http://a/1.htm"}, {"url": "http://a/2.htm", "failure": "http_status", "detail": "HTTP 403"}, + { + "url": "http://a/3.htm", + "failure": "unreachable", + "detail": "http://a/files/application.pdf", + }, ] def test_a_list_that_cant_be_read_ends_the_round_with_the_reason() -> None: - async def unreachable(_fetcher: Fetcher, source: WorkSource) -> list[ListedItem]: + async def unreachable(_pages: PageSession, source: WorkSource) -> list[ListedItem]: raise FetchError("unreachable", source.url) - async def empty(_fetcher: Fetcher, _source: WorkSource) -> list[ListedItem]: + async def empty(_pages: PageSession, _source: WorkSource) -> list[ListedItem]: return [] async def unused(*_: object) -> Any: @@ -204,3 +263,86 @@ async def unused(*_: object) -> Any: [run] = site.called("POST /runs") assert run["failure"] == failure assert run["listed"] == 0 + + +@pytest.fixture +def sessions( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[list[tuple[bool, PageSession]], list[PageSession]]: + entered: list[tuple[bool, PageSession]] = [] + exited: list[PageSession] = [] + original_session = Fetcher.session + + @asynccontextmanager + async def session(fetcher: Fetcher, browser: bool) -> AsyncGenerator[PageSession]: + async with original_session(fetcher, browser) as pages: + entered.append((browser, pages)) + try: + yield pages + finally: + exited.append(pages) + + monkeypatch.setattr(Fetcher, "session", session) + return entered, exited + + +def test_a_round_shares_one_session_and_reports_the_rendered_notice( + monkeypatch: pytest.MonkeyPatch, + sessions: tuple[list[tuple[bool, PageSession]], list[PageSession]], +) -> None: + entered, exited = sessions + source = SOURCE.model_copy(update={"browser": True}) + site = FakeWebsite([pending(1, "fetch")], extracted=frozenset({1})) + rendered = "

合成的渲染正文

" + seen: list[PageSession] = [] + + async def get(pages: PageSession, url: str) -> Response: + assert not exited + seen.append(pages) + html = "" if url == source.url else rendered + return Response(url, 200, "text/html; charset=utf-8", html.encode()) + + async def listing(pages: PageSession, source: WorkSource) -> list[ListedItem]: + response = await pages.get(source.url) + assert "合成的列表" in response.body.decode() + return ITEMS + + async def read(pages: PageSession, _source: WorkSource, url: str) -> NoticePage: + response = await pages.get(url) + return NoticePage(html=response.body.decode(), text="合成的渲染正文", attachments=[]) + + monkeypatch.setattr(PageSession, "get", get) + site.run(Steps(listing, screen, read, extract), source) + + assert len(seen) == 2 + assert seen[0] is seen[1] + assert entered == [(True, seen[0])] + assert exited == [seen[0]] + [snapshot] = site.called("POST /notices/1/snapshot") + assert snapshot["html"] == rendered + assert snapshot["text"] == "合成的渲染正文" + + +@pytest.mark.parametrize("failure", [ValueError, asyncio.CancelledError]) +def test_a_session_exits_when_a_step_fails_or_is_cancelled( + sessions: tuple[list[tuple[bool, PageSession]], list[PageSession]], + failure: type[BaseException], +) -> None: + entered, exited = sessions + site = FakeWebsite([], extracted=frozenset()) + + async def interrupted(pages: PageSession, _source: WorkSource) -> list[ListedItem]: + assert pages is entered[0][1] + raise failure("interrupted") + + async def read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: + raise AssertionError + + with pytest.raises(failure, match="interrupted"): + site.run(Steps(interrupted, screen, read, extract)) + + assert len(entered) == 1 + assert exited == [entered[0][1]] + assert not site.calls + with pytest.raises(RuntimeError, match="Page session is closed"): + asyncio.run(entered[0][1].get(SOURCE.url)) diff --git a/crawler/src/crawler/crawl/website.py b/crawler/src/crawler/crawl/website.py index 2df1a7e..2cfbe41 100644 --- a/crawler/src/crawler/crawl/website.py +++ b/crawler/src/crawler/crawl/website.py @@ -52,9 +52,10 @@ async def snapshot(self, notice_id: int, snapshot: Snapshot) -> SnapshotResult: body = await self._call("POST", f"/notices/{notice_id}/snapshot", snapshot) return SnapshotResult.model_validate_json(body) - async def attachment(self, snapshot_id: int, position: int, data: bytes) -> None: + async def attachment(self, snapshot_id: int, position: int, data: bytes, *, url: str) -> None: response = await self._client.put( f"/snapshots/{snapshot_id}/attachments/{position}", + params={"url": url}, content=data, headers={"Content-Type": "application/octet-stream"}, ) diff --git a/docs/api.md b/docs/api.md index e78d8bb..57e1728 100644 --- a/docs/api.md +++ b/docs/api.md @@ -101,13 +101,15 @@ | 接口 | 说明 | | --- | --- | | `GET /crawler/work` | 领活：到期要抓的来源和配置，管理员要试抓的地址。发出去的活带租约，到期前不再发；抓取节点中途断了，租约过期后重新发 | -| `POST /crawler/sources/{id}/listing` | 交来源列表页上的条目（网址、标题、日期）。返回这个来源要接着处理的通知：没初筛的；要抓正文的，也就是初筛通过、还没抽取的，以及项目还没截止、一天没抓过的（[总体设计 §7.4](design.md#74-发现变化)） | +| `POST /crawler/sources/{id}/listing` | 交来源列表页上的条目（网址、标题、日期）。返回这个来源要接着处理的通知：没初筛的；要抓正文的，也就是初筛通过但最新快照还没抽取的，以及项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的（[总体设计 §7.4](design.md#74-发现变化)） | | `POST /crawler/sources/{id}/screening` | 交一批通知的初筛分数和是否通过，带模型用量 | -| `POST /crawler/notices/{id}/snapshot` | 交抓到的正文、附件的名称、地址和文字。文字和上一份快照相同就不存新的。返回快照的编号和还缺文件的附件；要抽取时（有了新快照，或者上一份还没抽取），再附上抽取要对照的东西：通知的发布日期、这所学校在院校清单里的学院、这所学校发布那年到后两年这几届的项目（编号、学院、届、类型、名称、报名时间） | -| `PUT /crawler/snapshots/{id}/attachments/{position}` | 上传一个附件的文件，请求体就是文件本身 | +| `POST /crawler/notices/{id}/snapshot` | 交抓到的正文、附件的名称、地址和文字，附件下载失败时附上原因。文字和上一份快照相同就不存新的，下载错误等元数据仍更新。返回快照的编号和还缺文件的附件；要抽取时（有了新快照，或者上一份还没抽取），再附上抽取要对照的东西：通知的发布日期、这所学校在院校清单里的学院、这所学校发布那年到后两年这几届的项目（编号、学院、届、类型、名称、报名时间） | +| `PUT /crawler/snapshots/{id}/attachments/{position}` | 上传一个附件的文件，请求体就是文件本身，查询参数带原下载链接；链接已更新时拒绝旧文件上传 | | `POST /crawler/snapshots/{id}/extraction` | 交抽取的结果，带模型用量。结果是通知涉及的项目，每个带编号（新建的没有）、学院、届、类型、名称、作用和字段，格式见[总体设计 §8.2](design.md#82-抽取)；网站按 §8.3、§8.4 校验和归并 | | `POST /crawler/notices/{id}/removed` | 通知打不开（404、410），标成原文已删除；之后又能打开的，交新快照时取消 | | `POST /crawler/runs` | 交一个来源这一轮的结果：开始和结束时间、成功还是失败和原因、列表里几条、新通知几条、其中有用的几条、哪些通知没抓成和原因。交了才算这一轮结束，失败了也等下一轮再抓 | | `POST /crawler/probes/{id}` | 交试抓的结果：抓到的条目和生成的配置，或者失败的原因，带模型用量 | 一个来源的一轮：领活 → 抓列表页、交列表 → 对没初筛的初筛、交分数 → 对要抓正文的抓正文和附件、交快照、上传附件 → 要抽取的抽取、交结果 → 交这一轮的结果，没抓成的通知和原因写在里面。重抓正文也在来源的这一轮里，守同样的频率，失败了一样记进抓取记录。 + +单个附件下载失败仍保留名称、原下载链接和顺序，附件文字留空，原因写入 `error`：验证码明确标注需要验证码，接口的短错误提示照原文保存，取不到提示时记录 HTTP 状态或传输失败原因。网站不要求上传这些附件；最新快照仍有下载错误或缺文件的通知，即使抽取没有项目，也按一天一次的间隔在来源后续轮次里重抓，下载成功后清除错误并补传文件。同位置、同链接已有的文字和原件在下载失败时保留；仅下载错误变化不创建新快照、不重复抽取。文字不变而下载链接变化时刷新元数据，清除原链接的文件，等待补传。 diff --git a/docs/data-model.md b/docs/data-model.md index 2ba1c2d..c259e1a 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -36,7 +36,7 @@ | `crawl_runs` | `ingest` | 一个来源每一轮的结果，抓取看板用：`failure` 为空就是列表读到了，`detail` 写具体原因；各项计数；`notice_failures` 是没抓成的通知和原因 | | `notices` | `ingest` | 列表里的一条通知，`(source_id, url)` 唯一。`screening_score` 和 `screened_in` 是初筛的分数和结论，为空就是还没初筛；`checked_at` 是上次抓正文的时间，重抓用；`removed_at` 有值是原文已删除。抽取出项目的才有版本，没有版本的就是无关 | | `snapshots` | `ingest` | 某次抓到的正文，文字和上一份相同就不存新的；`extracted_at` 为空说明还没交抽取的结果 | -| `attachments` | `ingest` | 快照里的附件：名称、地址、类型、提取出的文字和文件本身。`position` 是在页面上的顺序，证据里的 `attachment` 指的就是它；文件在抓取节点上传后才有 | +| `attachments` | `ingest` | 快照里的附件：名称、地址、类型、提取出的文字和文件本身。`position` 是在页面上的顺序，证据里的 `attachment` 指的就是它；文件在抓取节点上传后才有。`error` 记录下载失败原因，成功时为空，和原文文字分开保存 | | `model_calls` | `ingest` | 判别模型和 LLM 的用量，随产出的结果一起交来 | ## 项目 diff --git a/docs/design.md b/docs/design.md index ebfbbd2..0a495e4 100644 --- a/docs/design.md +++ b/docs/design.md @@ -160,7 +160,7 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 ### 7.3 频率和礼貌 - 平时每个来源每天抓 2 次；3–9 月和 12–1 月是旺季，每 4 小时一次。 -- 同一个网站同一时间只发一个请求，两次请求之间隔 5 秒；请求头写明是鼠钟，附上联系方式。 +- 同一个网站同时只进行一次普通请求、附件下载或完整的浏览器页面访问；一次完成后等 5 秒，再开始下一次。浏览器的一次访问包含同站的脚本挑战、重载和页面子资源，子资源按浏览器正常方式加载；跨站跳转的排队规则见[开发约定 §2](development.md#2-抓取和判别)。请求头写明是鼠钟，附上联系方式。 - 不读 robots.txt：抓的都是公开的通知列表，礼貌靠上面这几条保证；有的学院官网一请求 robots.txt 就断开连接（比如西电），照它的规矩反而一页都抓不了。 - 失败了等下一轮再抓，一轮里不反复重试。 @@ -169,7 +169,7 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 - **新通知**：列表里出现没见过的链接，直接比网址。 - **正文变化**：还没截止的项目，每天重抓一次它的通知，在它的来源那一轮里一起抓；正文的文字和上一份快照不同，就存一份新快照，再交给抽取。 - **删除**：重抓时通知打不开（404、410），标成「原文已删除」，快照照样能看。从列表里消失不算：列表会翻页，旧通知本来就会被挤下去。 -- **附件**：PDF、Word、Excel 和图片都下载下来，解析出文字，图片做文字识别；附件和正文一起存进快照。 +- **附件**：PDF、Word、Excel 和图片都下载下来，解析出文字，图片做文字识别；附件和正文一起存进快照。单个附件下载失败不影响通知：保留名称、原下载链接和失败原因；需要验证码的明确标注，不破解。接口返回的短错误提示作为原因保存，不作为附件正文或文件。最新快照仍有下载错误或缺文件的通知，即使抽取没有项目，也在来源后续轮次里按一天一次的间隔重抓。 ### 7.5 抓取看板 diff --git a/docs/development.md b/docs/development.md index e8b195f..e957a29 100644 --- a/docs/development.md +++ b/docs/development.md @@ -59,13 +59,29 @@ crawler/ 抓取节点 - 接入来源时按通用抓取 → 配置 → 浏览器抓取 → 专用适配器的顺序试，前一种能抓就不用后一种（[总体设计 §7.2](design.md#72-怎么抓)）。 - 建站系统的模板放在代码里，和它的测试在一起；单个来源的配置存在网站的数据库里，在管理页面改，抓取节点领活时一起拿到。 -- 所有对来源网站的请求都经过 `crawl/fetch.py` 的 `Fetcher`，整个进程共用一个，[总体设计 §7.3](design.md#73-频率和礼貌)的规矩由它保证：UA 写明鼠钟；同一个网站一次一个请求，多个任务同时抓也一样；跳转的每一步单独发，跳到别的网站按那个网站的间隔排队。浏览器抓取打开页面前，也要先过这些规矩。 +- 所有对来源网站的访问都经过 `crawl/fetch.py` 的 `Fetcher`，整个进程共用一个，[总体设计 §7.3](design.md#73-频率和礼貌)的规矩由它保证：UA 写明鼠钟；普通 HTTP 请求、完整的浏览器页面访问和程序下载的附件共用按网站排队的许可，多个任务同时抓也一样；上一次访问完成并释放许可后，再等 5 秒才开始下一次。 +- 浏览器的 `get(url)` 在顶层导航前取得网站许可，持有到页面关闭，覆盖同站的 HTTP 跳转、脚本挑战和重载；页面子资源按浏览器正常方式加载。顶层导航跨 host 时，先释放原站许可，再等待目标站的许可。普通 HTTP 请求和附件的跳转每一步单独取得许可，跳到别的网站按那个网站的间隔排队。 +- `crawl_source` 每轮通过 `fetcher.session(source.browser)` 创建一个 `PageSession`，交给 `Steps.list_items` 和 `Steps.read`：`get(url)` 取页面，`download(url, page_url=原文网址)` 下载附件，两者都返回 `Response`。普通来源直接请求；`browser` 为真的来源用浏览器取渲染后的 HTML，以 UTF-8 字节返回。列表、正文和附件的解析仍由对应步骤负责。 +- 读取步骤用 `crawl/attachments.py` 的 `download_attachment` 下载附件：识别验证码、HTTP 和传输错误、接口返回的 HTML 错误提示，失败时返回原链接和独立的 `error`，文件为空；成功时保留原始字节，文字交给附件解析。错误页不上传为文件，错误原因不进入抽取用的 `text`，也不把整条通知标成已删除。日志只记下载失败和链接，不写源站返回的文字。 - 浏览器抓取启动时关掉 Chromium 的自动化标记（`--disable-blink-features=AutomationControlled`），UA 用浏览器自己的，把 `HeadlessChrome` 写成 `Chrome`，末尾加上鼠钟的标识；瑞数这类脚本挑战两处都检查（[总体设计 §5.2](design.md#52-实测)）。不冒充别的浏览器和系统。 -- 一轮里同一个网站共用一个无痕会话，抓完就关：脚本挑战设的 Cookie 在会话里有效，后面的页面不用再过挑战；不同网站、不同轮次之间不共享 Cookie。同时开的浏览器页面有上限，多的排队；拒绝弹窗和下载，附件由程序另外下载；不加载图片、字体和音视频，省对方的流量，也省内存。 +- 浏览器来源每一轮共用一个无痕会话，列表、通知、跨 host 跳转和附件沿用它，抓完就关，失败或取消也关闭；不同来源、不同轮次不共享会话。Cookie 由浏览器按 domain、path 和 secure 规则筛选；附件使用原文所在会话的请求客户端下载，共享浏览器管理的 Cookie。 +- 整个进程共用一个 Chromium 无头精简版，首次读取浏览器页面时才启动；同时最多开 2 个页面，多的排队。拒绝弹窗和浏览器下载；不加载图片、字体和音视频，省对方的流量，也省内存。 +- 浏览器页面等到最终文档已提交并触发 `DOMContentLoaded` 后读取 HTML，并确认读取期间未开始新的主文档导航。快照对应这一时刻的 DOM，列表和正文是否有效由解析步骤检查。 +- 普通 HTTP 请求（含普通来源的附件）使用 httpx 的网络操作超时：连接、连接池等待、每次读取和写入最多各等 30 秒，不是整次传输的总期限。浏览器来源的附件每个跳转请求总期限为 30 秒，从取得许可后开始，不含等待站点许可和间隔。 +- 浏览器页面总期限为 300 秒，从取得页面名额后开始，包含页面创建、站点许可排队、脚本挑战刷新和 DOM 就绪；等待页面名额不计入。期限内等不到最终页面就按超时记录，不把最初的 202、412 挑战页当成成功结果。成功、失败或取消后都先回收已发出的创建操作及页面，再释放网站许可和页面名额。 - 解析和模型分工：`parse` 用程序取出能确定的东西，模型只做判断（[总体设计 §8.2](design.md#82-抽取)）。解析不猜：判断不了的交给模型或人，不硬配。 - 判别模型和 LLM 的提示词放在调用它的代码旁边。输出都按固定的结构解析，解析不了的当失败处理。 - 每一轮抓取都交一份抓取记录，失败也交，写明原因；不在一轮里反复重试。各步抓不成时抛 `FetchError`（`Fetcher` 抛的）或 `StepError`（写明失败类别，比如 `http_status` 和「HTTP 403」），一轮的流程把它记进抓取记录；别的异常是 bug，照常抛出。 +本地使用浏览器抓取时，在 `crawler/` 运行： + +```sh +uv sync +uv run playwright install --with-deps --only-shell chromium +``` + +第二条安装 Chromium 无头精简版及系统依赖。CI 不安装浏览器：自动测试使用假的浏览器事件、固定响应和假时钟，不连接真实网站；真实浏览器和来源网站的验证在本地进行，结果写进 PR。 + ### 2.1 列表配置 特殊列表通过 `ListConfig` 校验 `WorkSource.config`，管理员与 #46 的试抓生成使用同一格式。例如只能用 HTTP 的浙大计算机学院列表： diff --git a/web/db/migrations/0001_init.sql b/web/db/migrations/0001_init.sql index 257a762..04f2329 100644 --- a/web/db/migrations/0001_init.sql +++ b/web/db/migrations/0001_init.sql @@ -141,6 +141,7 @@ CREATE TABLE attachments ( url text NOT NULL, content_type text NOT NULL, text text NOT NULL, -- extracted, or recognised in images + error text, -- why the attachment could not be downloaded data bytea, -- NULL until the crawler uploads the file PRIMARY KEY (snapshot_id, position) ); diff --git a/web/db/types.ts b/web/db/types.ts index b8a7cce..83832fe 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -26,6 +26,7 @@ export type Timestamp = ColumnType; export interface Attachments { content_type: string; data: Buffer | null; + error: string | null; name: string; position: number; snapshot_id: number; diff --git a/web/openapi/crawler.json b/web/openapi/crawler.json index 0ed5c1c..83a94a7 100644 --- a/web/openapi/crawler.json +++ b/web/openapi/crawler.json @@ -191,7 +191,7 @@ "screen", "fetch" ], - "description": "screen：还没初筛；fetch：要抓正文，初筛通过还没抽取的，或者项目还没截止、一天没抓过的" + "description": "screen：还没初筛；fetch：初筛通过但最新快照还没抽取的，或者项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的" } }, "required": [ @@ -600,6 +600,12 @@ "text": { "type": "string", "maxLength": 2097152 + }, + "error": { + "type": "string", + "minLength": 1, + "maxLength": 2000, + "description": "附件下载失败原因；不写表示下载正常，不影响通知正文和其他附件" } }, "required": [ @@ -1552,6 +1558,16 @@ "required": false, "name": "position", "in": "path" + }, + { + "schema": { + "type": "string", + "maxLength": 2000, + "format": "uri" + }, + "required": true, + "name": "url", + "in": "query" } ], "requestBody": { diff --git a/web/server/modules/ingest/routes.test.ts b/web/server/modules/ingest/routes.test.ts index e91771a..8a683d8 100644 --- a/web/server/modules/ingest/routes.test.ts +++ b/web/server/modules/ingest/routes.test.ts @@ -1,3 +1,4 @@ +import type { InferRequestType } from "hono/client"; import { testClient } from "hono/testing"; import { beforeAll, expect, test } from "vitest"; import { app } from "../../app.ts"; @@ -5,6 +6,15 @@ import { db } from "../../platform/db.ts"; import { createCrawlerToken } from "./service.ts"; const api = testClient(app).api.v1.crawler; +type AttachmentInfo = InferRequestType< + (typeof api.notices)[":id"]["snapshot"]["$post"] +>["json"]["attachments"][number]; +const file = { + name: "申请表.docx", + url: "https://x.edu.cn/f.docx", + content_type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + text: "表", +}; let crawler = { headers: {} }; @@ -45,25 +55,15 @@ async function list(sourceId: number, items: { url: string; title: string }[]) { return res.json(); } -async function snapshot(noticeId: number, attachmentText?: string) { +async function snapshot(noticeId: number, attachments: AttachmentInfo[] = [], text = "正文") { const res = await api.notices[":id"].snapshot.$post( { param: { id: String(noticeId) }, json: { fetched_at: 1780000000, html: "

正文

", - text: "正文", - attachments: - attachmentText === undefined - ? [] - : [ - { - name: "申请表.docx", - url: "https://x.edu.cn/f.docx", - content_type: "x", - text: attachmentText, - }, - ], + text, + attachments, }, }, crawler, @@ -222,10 +222,289 @@ test("the same words make no new snapshot, changed attachment text does", async const { pending } = await list(source.id, [{ url: "https://c.edu.cn/1.htm", title: "推免通知" }]); const noticeId = pending[0].id; - const first = await snapshot(noticeId, "表"); + const first = await snapshot(noticeId, [file]); expect(first).toMatchObject({ missing_files: [0], extraction: { colleges: [{}] } }); - expect(await snapshot(noticeId, "表")).toEqual(first); - expect((await snapshot(noticeId, "新表")).snapshot_id).not.toBe(first.snapshot_id); + expect(await snapshot(noticeId, [file])).toEqual(first); + expect((await snapshot(noticeId, [{ ...file, text: "新表" }])).snapshot_id).not.toBe( + first.snapshot_id, + ); +}); + +test("attachment download errors keep their links and do not require files", async () => { + const source = await newSource("attachments-error.edu.cn"); + const [notice] = ( + await list(source.id, [{ url: "https://attachments-error.edu.cn/1.htm", title: "通知" }]) + ).pending; + const attachments = [ + { ...file, text: "", error: "需要验证码" }, + { ...file, url: "https://x.edu.cn/deleted.docx", text: "", error: "获取文件错误" }, + ]; + const first = await snapshot(notice.id, attachments); + expect(first).toMatchObject({ missing_files: [], extraction: { colleges: [{}] } }); + expect(await snapshot(notice.id, attachments)).toEqual(first); + expect( + await db.selectFrom("snapshots").select("id").where("notice_id", "=", notice.id).execute(), + ).toHaveLength(1); + const stored = await db + .selectFrom("attachments") + .select(["position", "url", "text", "error", "data"]) + .where("snapshot_id", "=", first.snapshot_id) + .orderBy("position") + .execute(); + expect(stored).toEqual( + attachments.map((attachment, position) => ({ + position, + url: attachment.url, + text: "", + error: attachment.error, + data: null, + })), + ); +}); + +test("error updates and recovery reuse a text snapshot and request recovered files", async () => { + const source = await newSource("attachments-recovery.edu.cn"); + const [notice] = ( + await list(source.id, [{ url: "https://attachments-recovery.edu.cn/1.htm", title: "通知" }]) + ).pending; + const attachment = { ...file, text: "" }; + const first = await snapshot(notice.id, [{ ...attachment, error: "需要验证码" }]); + await db + .updateTable("snapshots") + .set({ extracted_at: new Date() }) + .where("id", "=", first.snapshot_id) + .execute(); + expect(await snapshot(notice.id, [{ ...attachment, error: "HTTP 503" }])).toEqual({ + snapshot_id: first.snapshot_id, + missing_files: [], + extraction: null, + }); + expect( + await db + .selectFrom("attachments") + .select("error") + .where("snapshot_id", "=", first.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ error: "HTTP 503" }); + expect(await snapshot(notice.id, [attachment])).toEqual({ + snapshot_id: first.snapshot_id, + missing_files: [0], + extraction: null, + }); + expect( + await db + .selectFrom("attachments") + .select("error") + .where("snapshot_id", "=", first.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ error: null }); + await db + .updateTable("attachments") + .set({ data: Buffer.from("original file") }) + .where("snapshot_id", "=", first.snapshot_id) + .execute(); + expect(await snapshot(notice.id, [attachment])).toEqual({ + snapshot_id: first.snapshot_id, + missing_files: [], + extraction: null, + }); +}); + +test("failed downloads preserve known attachment text and files without changing old snapshots", async () => { + const source = await newSource("attachments-known.edu.cn"); + const [notice] = ( + await list(source.id, [{ url: "https://attachments-known.edu.cn/1.htm", title: "通知" }]) + ).pending; + const first = await snapshot(notice.id, [file]); + const data = Buffer.from("original file"); + await db + .updateTable("attachments") + .set({ data }) + .where("snapshot_id", "=", first.snapshot_id) + .execute(); + await db + .updateTable("snapshots") + .set({ extracted_at: new Date() }) + .where("id", "=", first.snapshot_id) + .execute(); + const failed = { ...file, text: "", content_type: "text/html", error: "需要验证码" }; + expect(await snapshot(notice.id, [failed])).toEqual({ + snapshot_id: first.snapshot_id, + missing_files: [], + extraction: null, + }); + expect( + await db + .selectFrom("attachments") + .select(["text", "content_type", "error", "data"]) + .where("snapshot_id", "=", first.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ text: file.text, content_type: file.content_type, error: failed.error, data }); + const next = await snapshot(notice.id, [{ ...failed, error: "HTTP 503" }], "修改后的正文"); + expect(next.snapshot_id).not.toBe(first.snapshot_id); + expect(next.missing_files).toEqual([]); + expect( + await db + .selectFrom("attachments") + .select(["text", "error", "data"]) + .where("snapshot_id", "=", next.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ text: file.text, error: "HTTP 503", data }); + expect( + await db + .selectFrom("attachments") + .select("error") + .where("snapshot_id", "=", first.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ error: failed.error }); +}); + +test("attachment URL and order updates keep text versions and discard mismatched files", async () => { + const source = await newSource("attachments-order.edu.cn"); + const [notice] = ( + await list(source.id, [{ url: "https://attachments-order.edu.cn/1.htm", title: "通知" }]) + ).pending; + const attachments = [ + { ...file, text: "" }, + { ...file, url: "https://x.edu.cn/g.docx", text: "" }, + ]; + const first = await snapshot(notice.id, attachments); + await db + .updateTable("attachments") + .set({ data: Buffer.from("original file") }) + .where("snapshot_id", "=", first.snapshot_id) + .execute(); + const reordered = [{ ...attachments[1], error: "需要验证码" }, attachments[0]]; + expect(await snapshot(notice.id, reordered)).toMatchObject({ + snapshot_id: first.snapshot_id, + missing_files: [1], + }); + expect( + await db + .selectFrom("attachments") + .select(["position", "url", "error", "data"]) + .where("snapshot_id", "=", first.snapshot_id) + .orderBy("position") + .execute(), + ).toEqual( + reordered.map((attachment, position) => ({ + position, + url: attachment.url, + error: "error" in attachment ? attachment.error : null, + data: null, + })), + ); +}); + +test("a delayed attachment upload cannot overwrite a replacement URL", async () => { + const source = await newSource("attachments-stale.edu.cn"); + const [notice] = ( + await list(source.id, [{ url: "https://attachments-stale.edu.cn/1.htm", title: "通知" }]) + ).pending; + const original = { ...file, text: "" }; + const first = await snapshot(notice.id, [original]); + const replacement = { ...original, url: "https://x.edu.cn/replacement.docx" }; + expect(await snapshot(notice.id, [replacement])).toMatchObject({ + snapshot_id: first.snapshot_id, + missing_files: [0], + }); + const upload = (url: string, data: string) => + app.request( + `/api/v1/crawler/snapshots/${first.snapshot_id}/attachments/0?${new URLSearchParams({ url })}`, + { + method: "PUT", + headers: { ...crawler.headers, "content-type": "application/octet-stream" }, + body: data, + }, + ); + expect((await upload(original.url, "stale file")).status).toBe(404); + expect( + await db + .selectFrom("attachments") + .select("data") + .where("snapshot_id", "=", first.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ data: null }); + expect((await upload(replacement.url, "replacement file")).status).toBe(204); + expect( + await db + .selectFrom("attachments") + .select("data") + .where("snapshot_id", "=", first.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ data: Buffer.from("replacement file") }); +}); + +test("failed attachments are fetched once a day even when extraction found no programs", async () => { + const source = await newSource("attachments-retry.edu.cn"); + const items = [{ url: "https://attachments-retry.edu.cn/1.htm", title: "通知" }]; + const [notice] = (await list(source.id, items)).pending; + await api.sources[":id"].screening.$post( + { + param: { id: String(source.id) }, + json: { results: [{ notice_id: notice.id, score: 0.9, screened_in: true }], usage: [] }, + }, + crawler, + ); + const failed = { ...file, text: "", error: "HTTP 503" }; + const first = await snapshot(notice.id, [failed]); + const extracted = await api.snapshots[":id"].extraction.$post( + { param: { id: String(first.snapshot_id) }, json: { programs: [], usage: [] } }, + crawler, + ); + expect(extracted.status).toBe(204); + expect((await list(source.id, items)).pending).toEqual([]); + + const dayAgo = new Date(Date.now() - 25 * 3600_000); + await db.updateTable("notices").set({ checked_at: dayAgo }).where("id", "=", notice.id).execute(); + expect((await list(source.id, items)).pending).toMatchObject([{ id: notice.id, step: "fetch" }]); + + const recovered = await snapshot(notice.id, [{ ...file, text: "" }]); + expect(recovered).toEqual({ + snapshot_id: first.snapshot_id, + missing_files: [0], + extraction: null, + }); + await db.updateTable("notices").set({ checked_at: dayAgo }).where("id", "=", notice.id).execute(); + expect((await list(source.id, items)).pending).toMatchObject([{ id: notice.id, step: "fetch" }]); + const uploaded = await app.request( + `/api/v1/crawler/snapshots/${first.snapshot_id}/attachments/0?${new URLSearchParams({ url: file.url })}`, + { + method: "PUT", + headers: { ...crawler.headers, "content-type": "application/octet-stream" }, + body: "recovered file", + }, + ); + expect(uploaded.status).toBe(204); + expect((await list(source.id, items)).pending).toEqual([]); + + await snapshot(notice.id, [failed]); + const next = await snapshot(notice.id, [file]); + expect(next.snapshot_id).not.toBe(first.snapshot_id); + const nextUploaded = await app.request( + `/api/v1/crawler/snapshots/${next.snapshot_id}/attachments/0?${new URLSearchParams({ url: file.url })}`, + { + method: "PUT", + headers: { ...crawler.headers, "content-type": "application/octet-stream" }, + body: "new file", + }, + ); + expect(nextUploaded.status).toBe(204); + expect((await list(source.id, items)).pending).toMatchObject([{ id: notice.id, step: "fetch" }]); + const nextExtracted = await api.snapshots[":id"].extraction.$post( + { param: { id: String(next.snapshot_id) }, json: { programs: [], usage: [] } }, + crawler, + ); + expect(nextExtracted.status).toBe(204); + expect( + await db + .selectFrom("attachments") + .select("error") + .where("snapshot_id", "=", first.snapshot_id) + .executeTakeFirstOrThrow(), + ).toEqual({ error: failed.error }); + await db.updateTable("notices").set({ checked_at: dayAgo }).where("id", "=", notice.id).execute(); + expect((await list(source.id, items)).pending).toEqual([]); }); test("a notice of a program still open comes back to be fetched once a day", async () => { diff --git a/web/server/modules/ingest/routes.ts b/web/server/modules/ingest/routes.ts index 1dad207..bc03217 100644 --- a/web/server/modules/ingest/routes.ts +++ b/web/server/modules/ingest/routes.ts @@ -130,7 +130,7 @@ const ListingResult = z published_on: IsoDate.nullable(), step: z.enum(["screen", "fetch"]).openapi({ description: - "screen：还没初筛；fetch：要抓正文，初筛通过还没抽取的，或者项目还没截止、一天没抓过的", + "screen：还没初筛；fetch：初筛通过但最新快照还没抽取的，或者项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的", }), }) .openapi("PendingNotice"), @@ -287,6 +287,10 @@ export const ingestRoutes = router url: Url, content_type: z.string().min(1).max(200), text: z.string().max(2 * MB), + error: z.string().min(1).max(2000).optional().openapi({ + description: + "附件下载失败原因；不写表示下载正常，不影响通知正文和其他附件", + }), }) .openapi("AttachmentInfo"), ) @@ -323,6 +327,7 @@ export const ingestRoutes = router security, request: { params: z.object({ id: IntParam, position: IntParam }), + query: z.object({ url: Url }), body: { content: { "application/octet-stream": { schema: z.string().openapi({ format: "binary" }) }, @@ -338,7 +343,12 @@ export const ingestRoutes = router }), async (c) => { const { id, position } = c.req.valid("param"); - await storeAttachmentFile(id, position, Buffer.from(await c.req.arrayBuffer())); + await storeAttachmentFile( + id, + position, + c.req.valid("query").url, + Buffer.from(await c.req.arrayBuffer()), + ); return c.body(null, 204); }, ) diff --git a/web/server/modules/ingest/service.ts b/web/server/modules/ingest/service.ts index 3c9e39b..c201494 100644 --- a/web/server/modules/ingest/service.ts +++ b/web/server/modules/ingest/service.ts @@ -20,8 +20,8 @@ const LEASE_MS = 2 * 3600_000; const INTERVAL_MS = 12 * 3600_000; const BUSY_INTERVAL_MS = 4 * 3600_000; const BUSY_MONTHS = new Set([3, 4, 5, 6, 7, 8, 9, 12, 1]); -// Notices of programs still open are fetched again once a day to catch changes (design §7.4), -// in their source's round; those without a deadline, for this long after the program appeared. +// Incomplete attachments and notices of programs still open are fetched daily (design §7.4), +// in their source's round; programs without a deadline are watched for this long after creation. const RECHECK_MS = 24 * 3600_000; const UNDATED_RECHECK_MS = 90 * 24 * 3600_000; const MAX_SOURCES = 50; @@ -108,7 +108,7 @@ async function recordUsage( // Saves what the list showed and returns the source's notices that still need work: new ones // to screen; screened-in ones not yet extracted, which a round cut short leaves behind; and -// those of programs still open, not fetched for a day (design §7.4). +// those with incomplete attachments or programs still open, not fetched for a day (design §7.4). export async function recordListing(sourceId: number, items: ListedItem[]) { const now = new Date(); return db.transaction().execute(async (trx) => { @@ -131,44 +131,53 @@ export async function recordListing(sourceId: number, items: ListedItem[]) { .execute(); const pending = await trx .selectFrom("notices as n") + .leftJoinLateral( + (eb) => + eb + .selectFrom("snapshots") + .select(["id", "extracted_at"]) + .whereRef("notice_id", "=", "n.id") + .orderBy("id", "desc") + .limit(1) + .as("latest"), + (join) => join.onTrue(), + ) .select(["n.id", "n.url", "n.title", "n.published_on", "n.screening_score"]) .where("n.source_id", "=", sourceId) .where("n.removed_at", "is", null) .where((eb) => eb.or([ eb("n.screening_score", "is", null), + eb.and([eb("n.screened_in", "=", true), eb("latest.extracted_at", "is", null)]), eb.and([ - eb("n.screened_in", "=", true), - eb.not( + eb("n.checked_at", "<", new Date(now.getTime() - RECHECK_MS)), + eb.or([ eb.exists( eb .selectFrom("snapshots as s") - .select("s.id") + .innerJoin("program_versions as v", "v.snapshot_id", "s.id") + .innerJoin("programs as p", "p.id", "v.program_id") + .select("p.id") .whereRef("s.notice_id", "=", "n.id") - .where("s.extracted_at", "is not", null), - ), - ), - ]), - eb.and([ - eb("n.checked_at", "<", new Date(now.getTime() - RECHECK_MS)), - eb.exists( - eb - .selectFrom("snapshots as s") - .innerJoin("program_versions as v", "v.snapshot_id", "s.id") - .innerJoin("programs as p", "p.id", "v.program_id") - .select("p.id") - .whereRef("s.notice_id", "=", "n.id") - .where("p.deleted_at", "is", null) - .where((qb) => - qb.or([ - qb("p.apply_deadline", ">", now), - qb.and([ - qb("p.apply_deadline", "is", null), - qb("p.created_at", ">", new Date(now.getTime() - UNDATED_RECHECK_MS)), + .where("p.deleted_at", "is", null) + .where((qb) => + qb.or([ + qb("p.apply_deadline", ">", now), + qb.and([ + qb("p.apply_deadline", "is", null), + qb("p.created_at", ">", new Date(now.getTime() - UNDATED_RECHECK_MS)), + ]), ]), - ]), - ), - ), + ), + ), + eb.exists( + eb + .selectFrom("attachments as a") + .select("a.position") + .whereRef("a.snapshot_id", "=", "latest.id") + .where((qb) => qb.or([qb("a.error", "is not", null), qb("a.data", "is", null)])), + ), + ]), ]), ]), ) @@ -203,7 +212,13 @@ export async function recordScreening( }); } -type AttachmentSubmission = { name: string; url: string; content_type: string; text: string }; +type AttachmentSubmission = { + name: string; + url: string; + content_type: string; + text: string; + error?: string; +}; // A new snapshot only when the words changed, the attachments' included (design §7.4). export async function recordSnapshot( @@ -226,21 +241,28 @@ export async function recordSnapshot( .orderBy("id", "desc") .limit(1) .executeTakeFirst(); - const latestTexts = - latest && - ( - await trx + const latestAttachments = latest + ? await trx .selectFrom("attachments") - .select("text") + .select(["url", "content_type", "text"]) .where("snapshot_id", "=", latest.id) .orderBy("position") .execute() - ).map((a) => a.text); + : []; + const attachments = submission.attachments.map((attachment, position) => { + const previous = latestAttachments.at(position); + // A failed download tells us nothing new about the file's existing words or type. + return attachment.error !== undefined && + attachment.text === "" && + previous?.url === attachment.url + ? { ...attachment, text: previous.text, content_type: previous.content_type } + : attachment; + }); const unchanged = latest !== undefined && latest.text === submission.text && - latestTexts?.length === submission.attachments.length && - latestTexts.every((text, i) => text === submission.attachments[i].text); + latestAttachments.length === attachments.length && + latestAttachments.every((attachment, i) => attachment.text === attachments[i].text); const snapshot = unchanged ? latest @@ -254,17 +276,43 @@ export async function recordSnapshot( }) .returning(["id", "extracted_at"]) .executeTakeFirstOrThrow(); - if (!unchanged && submission.attachments.length > 0) { + if (unchanged) { + for (const [position, attachment] of attachments.entries()) { + await trx + .updateTable("attachments") + .set({ + name: attachment.name, + url: attachment.url, + content_type: attachment.content_type, + error: attachment.error ?? null, + ...(latestAttachments[position].url !== attachment.url ? { data: null } : {}), + }) + .where("snapshot_id", "=", snapshot.id) + .where("position", "=", position) + .execute(); + } + } else if (attachments.length > 0) { await trx .insertInto("attachments") .values( - submission.attachments.map((a, position) => ({ + attachments.map((a, position) => ({ snapshot_id: snapshot.id, position, ...a, })), ) .execute(); + if (latest) { + await sql` + UPDATE attachments AS current SET data = previous.data + FROM attachments AS previous + WHERE current.snapshot_id = ${snapshot.id} + AND previous.snapshot_id = ${latest.id} + AND current.position = previous.position + AND current.url = previous.url AND current.text = previous.text + AND current.error IS NOT NULL + `.execute(trx); + } } await trx .updateTable("notices") @@ -277,6 +325,7 @@ export async function recordSnapshot( .select("position") .where("snapshot_id", "=", snapshot.id) .where("data", "is", null) + .where("error", "is", null) .orderBy("position") .execute(); return { @@ -336,12 +385,18 @@ async function extractionContext( return { published_on: publishedOn, colleges, programs }; } -export async function storeAttachmentFile(snapshotId: number, position: number, data: Buffer) { +export async function storeAttachmentFile( + snapshotId: number, + position: number, + url: string, + data: Buffer, +) { const result = await db .updateTable("attachments") .set({ data }) .where("snapshot_id", "=", snapshotId) .where("position", "=", position) + .where("url", "=", url) .executeTakeFirst(); if (result.numUpdatedRows === 0n) throw new AppError(404, "not_found", "附件不存在"); } From cce0e190665a50d00143d3f2f855fa51e7805922 Mon Sep 17 00:00:00 2001 From: lym12321 <41183806+lym12321@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:49:04 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E7=AE=80=E5=8C=96=E6=B5=8F=E8=A7=88?= =?UTF-8?q?=E5=99=A8=E6=8A=93=E5=8F=96=E5=B9=B6=E7=A7=BB=E9=99=A4=E9=99=84?= =?UTF-8?q?=E4=BB=B6=E5=A4=B1=E8=B4=A5=E4=B8=8A=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/src/crawler/crawl/attachments.py | 115 -- crawler/src/crawler/crawl/attachments_test.py | 157 -- crawler/src/crawler/crawl/browser.py | 451 +----- crawler/src/crawler/crawl/browser_test.py | 1259 +++-------------- crawler/src/crawler/crawl/contract.py | 10 +- crawler/src/crawler/crawl/fetch.py | 43 +- crawler/src/crawler/crawl/fetch_test.py | 18 - crawler/src/crawler/crawl/rounds.py | 20 +- crawler/src/crawler/crawl/rounds_test.py | 151 +- crawler/src/crawler/crawl/website.py | 3 +- docs/api.md | 8 +- docs/data-model.md | 2 +- docs/design.md | 4 +- docs/development.md | 15 +- web/db/migrations/0001_init.sql | 1 - web/db/types.ts | 1 - web/openapi/crawler.json | 18 +- web/server/modules/ingest/routes.test.ts | 311 +--- web/server/modules/ingest/routes.ts | 14 +- web/server/modules/ingest/service.ts | 137 +- 20 files changed, 400 insertions(+), 2338 deletions(-) delete mode 100644 crawler/src/crawler/crawl/attachments.py delete mode 100644 crawler/src/crawler/crawl/attachments_test.py diff --git a/crawler/src/crawler/crawl/attachments.py b/crawler/src/crawler/crawl/attachments.py deleted file mode 100644 index 2dbcc32..0000000 --- a/crawler/src/crawler/crawl/attachments.py +++ /dev/null @@ -1,115 +0,0 @@ -import logging -import re -from dataclasses import dataclass -from html.parser import HTMLParser - -from crawler.crawl.contract import AttachmentInfo -from crawler.crawl.fetch import FetchError, PageSession, Response - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class Attachment: - info: AttachmentInfo - data: bytes | None - - -class _DownloadPage(HTMLParser): - def __init__(self) -> None: - super().__init__() - self.parts: list[str] = [] - self.hidden = 0 - self.office = False - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if tag == "html": - self.office = any( - (name == "xmlns" or name.startswith("xmlns:")) - and value is not None - and value.startswith("urn:schemas-microsoft-com:office:") - for name, value in attrs - ) - if tag in {"title", "script", "style"}: - self.hidden += 1 - - def handle_endtag(self, tag: str) -> None: - if tag in {"title", "script", "style"}: - self.hidden = max(0, self.hidden - 1) - - def handle_data(self, data: str) -> None: - if not self.hidden: - self.parts.append(data) - - -def _download_error(response: Response) -> str | None: - status = f"HTTP {response.status}" if not 200 <= response.status < 300 else None - # Some download endpoints label Office and PDF files as HTML. - if status is None and response.body.startswith((b"%PDF-", b"\xd0\xcf\x11\xe0", b"PK\x03\x04")): - return None - content_type = response.content_type or "" - charset = re.search(r"charset\s*=\s*['\"]?([\w.-]+)", content_type, re.IGNORECASE) - meta = re.search( - rb"]*\bcharset\s*=\s*['\"]?([\w.-]+)", response.body[:1024], re.IGNORECASE - ) - if response.body.startswith(b"\xef\xbb\xbf"): - encoding = "utf-8-sig" - elif response.body.startswith((b"\xff\xfe", b"\xfe\xff")): - encoding = "utf-16" - elif charset: - encoding = charset[1] - elif meta: - encoding = meta[1].decode("ascii") - else: - encoding = "utf-8-sig" - try: - prefix = response.body[:1024].decode(encoding, errors="replace") - except LookupError: - encoding = "utf-8-sig" - prefix = response.body[:1024].decode(encoding, errors="replace") - html = content_type.split(";")[0].strip().lower() in {"text/html", "application/xhtml+xml"} - html = ( - html - or re.match( - r"<(?:!doctype\s+html|html|head|body|title|meta|script|style|form|div|p|span|pre|table|h[1-6])" - r"(?:\s|/?>)", - prefix.lstrip(), - re.IGNORECASE, - ) - is not None - ) - if not html: - return status or ("附件内容为空" if not response.body else None) - - page = _DownloadPage() - page.feed(response.body.decode(encoding, errors="replace")) - if status is None and page.office: - return None - text = " ".join(" ".join(page.parts).split()) - if "验证码" in text or "captcha" in text.lower(): - return "下载附件需要验证码" - # A whole notice or login page is not a useful attachment failure message. - detail = text if 0 < len(text) <= 1000 else "附件地址未返回文件" - return detail if status is None else f"{status}: {detail}" - - -async def download_attachment( - pages: PageSession, url: str, *, name: str, page_url: str -) -> Attachment: - content_type = "application/octet-stream" - data: bytes | None = None - try: - response = await pages.download(url, page_url=page_url) - except FetchError as failure: - error = failure.failure - else: - content_type = response.content_type or content_type - error = _download_error(response) - if error is None: - data = response.body - if error is not None: - logger.warning("Attachment download failed", extra={"url": url}) - return Attachment( - AttachmentInfo(name=name, url=url, content_type=content_type, text="", error=error), - data, - ) diff --git a/crawler/src/crawler/crawl/attachments_test.py b/crawler/src/crawler/crawl/attachments_test.py deleted file mode 100644 index 1b27116..0000000 --- a/crawler/src/crawler/crawl/attachments_test.py +++ /dev/null @@ -1,157 +0,0 @@ -import asyncio - -import httpx -import pytest - -from crawler.crawl.attachments import download_attachment -from crawler.crawl.fetch import Fetcher - -URL = "https://school.test/download.jsp?file=1" -PAGE_URL = "https://school.test/notice.htm" - - -@pytest.mark.parametrize( - ("body", "content_type", "status", "error"), - [ - ( - '
请输入验证码下载附件
', - "text/html;charset=UTF-8", - 200, - "下载附件需要验证码", - ), - ("
获取文件错误
", "text/html;charset=UTF-8", 200, "获取文件错误"), - ("

附件已删除

", "text/html;charset=gb2312", 200, "附件已删除"), - ( - '

文件已删除

', - "text/html;charset=gb2312", - 200, - "文件已删除", - ), - ( - "

文件已删除

".encode("utf-8-sig"), - "text/html;charset=gb2312", - 200, - "文件已删除", - ), - ( - ( - '请输入验证码下载附件' - ).encode("gb2312"), - "text/html", - 200, - "下载附件需要验证码", - ), - ( - ( - '' - "

文件已删除

" - ).encode("gb2312"), - "text/html", - 200, - "文件已删除", - ), - ( - "

请输入验证码下载附件

".encode("utf-16"), - None, - 200, - "下载附件需要验证码", - ), - ( - "下载请输入验证码下载附件", - "text/html;charset=UTF-8", - 200, - "下载附件需要验证码", - ), - ( - "下载

文件已删除

", - "text/html;charset=UTF-8", - 200, - "文件已删除", - ), - ("

文件不存在

", "text/html", 404, "HTTP 404: 文件不存在"), - ("", None, 200, "附件地址未返回文件"), - ( - "文件已删除", - "application/octet-stream", - 200, - "文件已删除", - ), - ("\ufeff\n
获取文件错误
", None, 200, "获取文件错误"), - ("", "application/pdf", 200, "附件内容为空"), - ("denied", "text/plain", 403, "HTTP 403"), - ], -) -def test_download_errors_keep_the_link_and_never_keep_an_error_page_as_a_file( - body: str | bytes, content_type: str | None, status: int, error: str -) -> None: - encoding = "gb2312" if content_type and "gb2312" in content_type else "utf-8" - content = body.encode(encoding) if isinstance(body, str) else body - headers = {"content-type": content_type} if content_type else {} - - async def main() -> None: - transport = httpx.MockTransport( - lambda _: httpx.Response(status, content=content, headers=headers) - ) - async with Fetcher(transport) as fetcher, fetcher.session(False) as pages: - attachment = await download_attachment(pages, URL, name="申请表.doc", page_url=PAGE_URL) - assert attachment.info.url == URL - assert attachment.info.name == "申请表.doc" - assert attachment.info.text == "" - assert attachment.info.error == error - assert attachment.data is None - - asyncio.run(main()) - - -@pytest.mark.parametrize( - ("body", "content_type"), - [ - (b"%PDF-1.7\n", "text/html"), - (b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", "text/html"), - (b"PK\x03\x04", "text/html"), - ( - '' - '' - '申请表' - "
".encode(), - "application/vnd.ms-excel", - ), - ( - '' - "申请表" - "
申请表
".encode(), - "application/vnd.ms-excel", - ), - ], -) -def test_downloads_keep_binary_and_office_files_after_redirects( - body: bytes, content_type: str -) -> None: - async def main() -> None: - def handle(request: httpx.Request) -> httpx.Response: - if request.url.path == "/download.jsp": - return httpx.Response(302, headers={"location": "https://files.test/form"}) - return httpx.Response(200, content=body, headers={"content-type": content_type}) - - async with Fetcher(httpx.MockTransport(handle)) as fetcher, fetcher.session(False) as pages: - attachment = await download_attachment(pages, URL, name="申请表", page_url=PAGE_URL) - assert attachment.info.url == URL - assert attachment.info.error is None - assert attachment.data == body - - asyncio.run(main()) - - -def test_a_transport_failure_is_an_attachment_error() -> None: - async def main() -> None: - def handle(request: httpx.Request) -> httpx.Response: - raise httpx.ReadTimeout("timed out", request=request) - - async with Fetcher(httpx.MockTransport(handle)) as fetcher, fetcher.session(False) as pages: - attachment = await download_attachment(pages, URL, name="申请表", page_url=PAGE_URL) - assert attachment.info.url == URL - assert attachment.info.error == "timeout" - assert attachment.data is None - - asyncio.run(main()) diff --git a/crawler/src/crawler/crawl/browser.py b/crawler/src/crawler/crawl/browser.py index 63ff05c..97603eb 100644 --- a/crawler/src/crawler/crawl/browser.py +++ b/crawler/src/crawler/crawl/browser.py @@ -2,16 +2,13 @@ import logging import time from collections.abc import Awaitable, Callable -from typing import Protocol -from urllib.parse import urljoin, urlsplit +from urllib.parse import urljoin from playwright.async_api import ( Browser as PlaywrightBrowser, ) from playwright.async_api import ( BrowserContext, - Download, - Frame, Page, Playwright, Route, @@ -20,16 +17,12 @@ from playwright.async_api import ( Error as PlaywrightError, ) -from playwright.async_api import ( - Request as PlaywrightRequest, -) from playwright.async_api import ( Response as PlaywrightResponse, ) from playwright.async_api import ( TimeoutError as PlaywrightTimeoutError, ) -from pydantic import BaseModel, Field, JsonValue, TypeAdapter, ValidationError from crawler.crawl.fetch import ( MAX_REDIRECTS, @@ -38,7 +31,6 @@ FetchError, Permit, Response, - site_key, ) PAGE_TIMEOUT = 300.0 @@ -46,80 +38,9 @@ _REDIRECTS = {301, 302, 303, 307, 308} _BLOCKED = {"image", "font", "media"} _CHALLENGES = {202, 412} -_PARAMS = TypeAdapter(dict[str, JsonValue]) _logger = logging.getLogger(__name__) -class _Version(BaseModel): - user_agent: str = Field(alias="userAgent") - - -class _Request(BaseModel): - url: str - - -class _Paused(BaseModel): - request_id: str = Field(alias="requestId") - request: _Request - frame_id: str = Field(alias="frameId") - redirected_request_id: str | None = Field(default=None, alias="redirectedRequestId") - - -class _Frame(BaseModel): - id: str - - -class _FrameTree(BaseModel): - frame: _Frame - - -class _Frames(BaseModel): - frame_tree: _FrameTree = Field(alias="frameTree") - - -# Playwright leaves CDPSession.send's dictionaries untyped; validate its JSON at this boundary. -class _Commands(Protocol): - async def send(self, method: str, params: dict[str, JsonValue] | None = None) -> object: ... - - def on(self, event: str, f: Callable[[object], None]) -> None: ... - - async def detach(self) -> None: ... - - -async def _command( - connection: _Commands, method: str, params: dict[str, JsonValue] | None = None -) -> dict[str, JsonValue]: - return _PARAMS.validate_python(await connection.send(method, params)) - - -def _source_error(error: PlaywrightError, url: str) -> FetchError | None: - if isinstance(error, PlaywrightTimeoutError): - return FetchError("timeout", url) - message = str(error).split("\n", 1)[0] - if "Download is starting" in message: - return FetchError("unreachable", url) - if "net::ERR_TOO_MANY_REDIRECTS" in message: - return FetchError("too_many_redirects", url) - if "net::ERR_TIMED_OUT" in message or "Request timed out" in message: - return FetchError("timeout", url) - if any( - marker in message - for marker in ("net::ERR_", "ECONN", "ENOTFOUND", "EAI_AGAIN", "socket hang up") - ): - return FetchError("unreachable", url) - if message.startswith( - ( - "APIRequestContext.get: self-signed certificate", - "APIRequestContext.get: certificate has expired", - "APIRequestContext.get: unable to verify the first certificate", - "APIRequestContext.get: Hostname/IP does not match certificate's altnames:", - "APIRequestContext.get: failed to decompress 'gzip' encoding:", - ) - ): - return FetchError("unreachable", url) - return None - - class Browser: def __init__(self, permit: Callable[[str], Awaitable[Permit]]) -> None: self.permit = permit @@ -128,7 +49,6 @@ def __init__(self, permit: Callable[[str], Awaitable[Permit]]) -> None: self._playwright: Playwright | None = None self._browser: PlaywrightBrowser | None = None self._user_agent = "" - self.sessions: set[BrowserSession] = set() async def _start(self) -> PlaywrightBrowser: async with self._lock: @@ -139,108 +59,91 @@ async def _start(self) -> PlaywrightBrowser: browser = await playwright.chromium.launch( headless=True, args=["--disable-blink-features=AutomationControlled"] ) - connection: _Commands = await browser.new_browser_cdp_session() + page = await browser.new_page() try: - version = _Version.model_validate( - await _command(connection, "Browser.getVersion") - ) + user_agent: str = await page.evaluate("navigator.userAgent") finally: - await connection.detach() + await page.close() except BaseException: await playwright.stop() raise - self._user_agent = ( - version.user_agent.replace("HeadlessChrome", "Chrome") + " " + USER_AGENT - ) + self._user_agent = user_agent.replace("HeadlessChrome", "Chrome") + " " + USER_AGENT self._playwright = playwright self._browser = browser return browser - async def session(self) -> BrowserSession: - browser = await self._start() - creating = asyncio.create_task( - browser.new_context( + async def session(self, url: str) -> BrowserSession: + try: + browser = await self._start() + context = await browser.new_context( user_agent=self._user_agent, service_workers="block", accept_downloads=False ) - ) - try: - # Cancelling a protocol call does not prevent the browser from creating its resource. - await asyncio.wait((creating,)) - context = creating.result() session = BrowserSession(self, context) - await session.configure() - except BaseException: - context = await creating - await context.close() - raise - self.sessions.add(session) - return session + try: + await session.configure() + except BaseException: + await context.close() + raise + return session + except PlaywrightTimeoutError as error: + raise FetchError("timeout", url) from error + except PlaywrightError as error: + raise FetchError("unreachable", url) from error async def close(self) -> None: try: - for session in tuple(self.sessions): - await session.close() + if self._browser is not None: + await self._browser.close() finally: - try: - if self._browser is not None: - await self._browser.close() - self._browser = None - finally: - if self._playwright is not None: - await self._playwright.stop() - self._playwright = None + if self._playwright is not None: + await self._playwright.stop() class BrowserSession: def __init__(self, owner: Browser, context: BrowserContext) -> None: self._owner = owner self._context = context - self._pages: set[Page] = set() - self._popup_tasks: set[asyncio.Task[None]] = set() - self._closed = False async def configure(self) -> None: await self._context.route("**/*", self._route) self._context.on("page", self._new_page) - def _new_page(self, page: Page) -> None: - task = asyncio.create_task(self._popup(page)) - self._popup_tasks.add(task) - - async def _popup(self, page: Page) -> None: + async def _new_page(self, page: Page) -> None: if await page.opener() is not None: await page.close() async def _route(self, route: Route) -> None: - request = route.request - if request.resource_type in _BLOCKED: + if route.request.resource_type in _BLOCKED: await route.abort() - return - try: - allowed = request.frame.page in self._pages - except PlaywrightError as error: - if "Frame for this navigation request is not available" not in str(error): - raise - allowed = False - if not allowed: - await route.abort() - return - await route.continue_() + else: + await route.continue_() async def get(self, url: str) -> Response: async with self._owner.slots: - page: Page | None = None - creating: asyncio.Task[Page] | None = None - request: _Page | None = None + permit = await self._owner.permit(url) started = time.monotonic() try: - async with asyncio.timeout(PAGE_TIMEOUT): - creating = asyncio.create_task(self._context.new_page()) - await asyncio.wait((creating,)) - page = creating.result() - self._pages.add(page) - request = _Page(page, self._context, self._owner.permit, url) - response = await request.run() + page = await self._context.new_page() + try: + last_response: PlaywrightResponse | None = None + + def record_response(response: PlaywrightResponse) -> None: + nonlocal last_response + request = response.request + if request.is_navigation_request() and request.frame == page.main_frame: + last_response = response + + page.on("response", record_response) + await page.goto(url, wait_until="networkidle", timeout=PAGE_TIMEOUT * 1000) + if last_response is None or last_response.status in _CHALLENGES: + raise FetchError("unreachable", page.url) + body = await page.content() + response = Response( + page.url, + last_response.status, + "text/html; charset=utf-8", + body.encode("utf-8"), + ) _logger.info( "Browser fetched %s status=%s elapsed=%.3fs", response.url, @@ -248,16 +151,14 @@ async def get(self, url: str) -> Response: time.monotonic() - started, ) return response - except TimeoutError as error: + finally: + await page.close() + except PlaywrightTimeoutError as error: raise FetchError("timeout", url) from error + except PlaywrightError as error: + raise FetchError("unreachable", url) from error finally: - if request is not None: - await request.close() - elif creating is not None: - page = await creating - await page.close() - if page is not None: - self._pages.discard(page) + permit.release() async def download(self, url: str, *, page_url: str) -> Response: for _ in range(MAX_REDIRECTS + 1): @@ -284,247 +185,13 @@ async def download(self, url: str, *, page_url: str) -> Response: ) finally: await response.dispose() + except PlaywrightTimeoutError as error: + raise FetchError("timeout", url) from error except PlaywrightError as error: - failure = _source_error(error, url) - if failure is not None: - raise failure from error - raise + raise FetchError("unreachable", url) from error finally: permit.release() raise FetchError("too_many_redirects", url) async def close(self) -> None: - if self._closed: - return - self._closed = True - try: - await self._context.close() - finally: - try: - if self._popup_tasks: - await asyncio.gather(*self._popup_tasks) - finally: - self._owner.sessions.discard(self) - - -class _Page: - def __init__( - self, - page: Page, - context: BrowserContext, - permit: Callable[[str], Awaitable[Permit]], - url: str, - ) -> None: - self._context = context - self._permit = permit - self._page = page - self._url = url - self._connection: _Commands | None = None - self._frame_id = "" - self._tasks: set[asyncio.Task[None]] = set() - self._gate = asyncio.Lock() - self._current_permit: Permit | None = None - self._site: str | None = None - self._redirects: dict[str, int] = {} - self._changed = asyncio.Event() - self._failure: asyncio.Future[BaseException] = asyncio.get_running_loop().create_future() - self._main: PlaywrightResponse | None = None - self._current_request: PlaywrightRequest | None = None - self._generation = 0 - self._committed_generation = -1 - self._loaded_generation = -1 - self._document_pending = False - self._closing = False - - def _spawn(self, operation: Awaitable[None]) -> None: - task = asyncio.ensure_future(operation) - self._tasks.add(task) - task.add_done_callback(self._done) - - def _done(self, task: asyncio.Task[None]) -> None: - self._tasks.discard(task) - if not task.cancelled(): - error = task.exception() - if error is not None: - self._fail(error) - self._changed.set() - - def _fail(self, error: BaseException) -> None: - if not self._closing and not self._failure.done(): - self._failure.set_result(error) - - async def _send( - self, method: str, params: dict[str, JsonValue] | None = None - ) -> dict[str, JsonValue]: - if self._connection is None: - raise RuntimeError("CDP connection has not started") - return await _command(self._connection, method, params) - - async def _configure(self) -> None: - frames = _Frames.model_validate(await self._send("Page.getFrameTree")) - self._frame_id = frames.frame_tree.frame.id - await self._send( - "Fetch.enable", - { - "patterns": [ - {"urlPattern": "*", "resourceType": "Document", "requestStage": "Request"} - ] - }, - ) - - def _request_paused(self, value: object) -> None: - if self._closing: - return - try: - event = _Paused.model_validate(value) - except ValidationError as error: - self._fail(error) - return - if event.frame_id == self._frame_id: - self._document_pending = True - self._changed.set() - self._spawn(self._paused(event)) - - def _release(self) -> None: - if self._current_permit is not None: - self._current_permit.release() - self._current_permit = None - self._site = None - - async def _paused(self, event: _Paused) -> None: - if event.frame_id == self._frame_id: - async with self._gate: - redirects = ( - self._redirects.pop(event.redirected_request_id, 0) + 1 - if event.redirected_request_id is not None - else 0 - ) - if redirects > MAX_REDIRECTS: - self._fail(FetchError("too_many_redirects", event.request.url)) - return - self._redirects[event.request_id] = redirects - url = event.request.url - if urlsplit(url).scheme in {"http", "https"}: - site = site_key(url) - if site != self._site: - # Release before waiting at the destination so two visits crossing - # between the same sites cannot hold each other's permits. - self._release() - permit = await self._permit(url) - if self._closing: - permit.release() - return - self._current_permit = permit - self._site = site - await self._send("Fetch.continueRequest", {"requestId": event.request_id}) - else: - await self._send("Fetch.continueRequest", {"requestId": event.request_id}) - - def _requested(self, request: PlaywrightRequest) -> None: - if request.is_navigation_request() and request.frame == self._page.main_frame: - self._current_request = request - self._generation += 1 - self._main = None - self._document_pending = True - self._changed.set() - - def _response(self, response: PlaywrightResponse) -> None: - if response.request is self._current_request: - self._main = response - self._changed.set() - - def _request_failed(self, request: PlaywrightRequest) -> None: - if request is not self._current_request: - return - reason = request.failure - if reason == "net::ERR_ABORTED": - return - error = PlaywrightError(reason or "Main navigation failed without a reason") - self._fail(_source_error(error, request.url) or error) - - def _navigated(self, frame: Frame) -> None: - if frame == self._page.main_frame: - # History/hash changes have no new response or DOMContentLoaded event. - if self._main is not None and self._committed_generation != self._generation: - self._committed_generation = self._generation - self._document_pending = False - self._changed.set() - - def _domcontentloaded(self, _: Page) -> None: - if self._committed_generation == self._generation: - self._loaded_generation = self._generation - self._changed.set() - - def _download(self, download: Download) -> None: - self._fail(FetchError("unreachable", download.url)) - self._spawn(download.cancel()) - - def _ready(self) -> bool: - return ( - self._main is not None - and self._committed_generation == self._generation - and self._loaded_generation == self._generation - and not self._document_pending - and not self._tasks - ) - - async def _navigate(self) -> Response: - await self._page.goto(self._url, wait_until="commit", timeout=0) - while True: - self._changed.clear() - response = self._main - if response is not None and response.status not in _CHALLENGES and self._ready(): - generation = self._generation - try: - body = await self._page.content() - except PlaywrightError as error: - if "page is navigating" not in str(error): - raise - continue - if generation == self._generation and response is self._main and self._ready(): - return Response( - self._page.url, - response.status, - "text/html; charset=utf-8", - body.encode("utf-8"), - ) - await self._changed.wait() - - async def run(self) -> Response: - self._connection = await self._context.new_cdp_session(self._page) - self._connection.on("Fetch.requestPaused", self._request_paused) - self._page.on("request", self._requested) - self._page.on("requestfailed", self._request_failed) - self._page.on("response", self._response) - self._page.on("download", self._download) - self._page.on("framenavigated", self._navigated) - self._page.on("domcontentloaded", self._domcontentloaded) - navigation: asyncio.Task[Response] | None = None - try: - await self._configure() - navigation = asyncio.create_task(self._navigate()) - await asyncio.wait((navigation, self._failure), return_when=asyncio.FIRST_COMPLETED) - if self._failure.done(): - raise self._failure.result() - return await navigation - except PlaywrightError as error: - failure = _source_error(error, self._page.url or self._url) - if failure is not None: - raise failure from error - raise - finally: - if navigation is not None and not navigation.done(): - navigation.cancel() - await asyncio.gather(navigation, return_exceptions=True) - - async def close(self) -> None: - self._closing = True - try: - await self._page.close() - finally: - tasks = tuple(self._tasks) - for task in tasks: - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - self._release() + await self._context.close() diff --git a/crawler/src/crawler/crawl/browser_test.py b/crawler/src/crawler/crawl/browser_test.py index 3298802..0574f87 100644 --- a/crawler/src/crawler/crawl/browser_test.py +++ b/crawler/src/crawler/crawl/browser_test.py @@ -1,22 +1,14 @@ import asyncio from collections.abc import Awaitable, Callable +from dataclasses import dataclass import httpx import pytest from playwright.async_api import Error as PlaywrightError from playwright.async_api import TimeoutError as PlaywrightTimeoutError -from pydantic import JsonValue from crawler.crawl.browser import Browser -from crawler.crawl.fetch import ( - INTERVAL, - MAX_REDIRECTS, - Failure, - Fetcher, - FetchError, - Permit, - Response, -) +from crawler.crawl.fetch import INTERVAL, Failure, Fetcher, FetchError def run(main: Callable[[], Awaitable[None]]) -> None: @@ -27,32 +19,6 @@ async def bounded() -> None: asyncio.run(bounded()) -class FakeSession: - def __init__(self, number: int) -> None: - self.number = number - self.closed = False - - async def get(self, url: str) -> Response: - return Response(url, 200, "text/html", f"session {self.number}".encode()) - - async def download(self, url: str, *, page_url: str) -> Response: - return Response(url, 200, "application/pdf", f"session {self.number}: {page_url}".encode()) - - async def close(self) -> None: - self.closed = True - - -class FakeSessions: - def __init__(self) -> None: - self.sessions: list[FakeSession] = [] - - async def session(self) -> FakeSession: - await asyncio.sleep(0) - session = FakeSession(len(self.sessions) + 1) - self.sessions.append(session) - return session - - class FakeClock: def __init__(self) -> None: self.now = 0.0 @@ -65,670 +31,277 @@ async def sleep(self, seconds: float) -> None: await asyncio.sleep(0) -class FakeAPIResponse: - def __init__(self, response: httpx.Response) -> None: - self.response = response - self.url = str(response.url) - self.status = response.status_code - self.headers = dict(response.headers) - self.disposed = False - - async def body(self) -> bytes: - return self.response.content - - async def dispose(self) -> None: - self.disposed = True - await self.response.aclose() - - -class FakeDownloads: - def __init__(self, client: httpx.AsyncClient) -> None: - self.client = client - self.responses: list[FakeAPIResponse] = [] - self.error: PlaywrightError | None = None - - async def get( - self, - url: str, - *, - headers: dict[str, str], - **options: object, - ) -> FakeAPIResponse: - if self.error is not None: - raise self.error - assert options["max_redirects"] == options["max_retries"] == 0 - assert options["fail_on_status_code"] is False - request_timeout = options["timeout"] - assert isinstance(request_timeout, int | float) - response = FakeAPIResponse( - await self.client.get(url, headers=headers, timeout=request_timeout / 1000) - ) - self.responses.append(response) - return response - - -class FakeCDP: - def __init__(self) -> None: - self.handlers: dict[str, Callable[..., None]] = {} - self.commands: asyncio.Queue[tuple[str, dict[str, JsonValue]]] = asyncio.Queue() - self.continued: dict[str, asyncio.Event] = {} - - def on(self, event: str, callback: Callable[..., None]) -> None: - self.handlers[event] = callback - - async def send( - self, method: str, params: dict[str, JsonValue] | None = None - ) -> dict[str, JsonValue]: - self.commands.put_nowait((method, params or {})) - if method == "Page.getFrameTree": - return {"frameTree": {"frame": {"id": "main"}}} - if method == "Fetch.continueRequest": - assert params is not None - request_id = params["requestId"] - assert isinstance(request_id, str) - self.continued[request_id].set() - return {} - - async def next_command( - self, method: str, *, request_id: str | None = None - ) -> dict[str, JsonValue]: - while True: - name, params = await self.commands.get() - if name == method and (request_id is None or params.get("requestId") == request_id): - return params - - def pause( - self, - request_id: str, - url: str, - *, - frame_id: str = "main", - redirected: str | None = None, - ) -> None: - self.continued[request_id] = asyncio.Event() - value: dict[str, JsonValue] = { - "requestId": request_id, - "request": {"url": url, "method": "GET"}, - "resourceType": "Document", - "frameId": frame_id, - } - if redirected is not None: - value["redirectedRequestId"] = redirected - self.handlers["Fetch.requestPaused"](value) - - -class FakeFrame: - def __init__(self, page: FakePage) -> None: - self.page = page - - -class FakeNavigation: - def __init__(self, frame: FakeFrame, url: str, *, navigation: bool = True) -> None: - self.frame = frame - self.url = url - self.navigation = navigation - self.failure: str | None = None +@dataclass +class FakeRequest: + frame: object + navigation: bool def is_navigation_request(self) -> bool: return self.navigation -class FakeDocument: - def __init__(self, status: int, request: FakeNavigation) -> None: - self.request = request - self.status = status +@dataclass +class FakeResponse: + status: int + request: FakeRequest class FakePage: def __init__(self, context: FakeContext) -> None: self.context = context - self.main_frame = FakeFrame(self) - self.navigation = FakeNavigation(self.main_frame, "about:blank") + self.main_frame = object() self.url = "about:blank" - self.html = "" self.closed = False - self.close_started = asyncio.Event() - self.close_gate: asyncio.Event | None = None - self.started = asyncio.Event() - self.content_started = asyncio.Event() - self.content_gate: asyncio.Event | None = None - self.navigation_error: PlaywrightError | None = None - self.connection = FakeCDP() - self.handlers: dict[str, Callable[..., None]] = {} - - def on(self, event: str, callback: Callable[..., None]) -> None: - self.handlers[event] = callback + self.ready = asyncio.Event() + self.handlers: dict[str, Callable[[FakeResponse], None]] = {} - def begin_navigation(self, url: str) -> FakeNavigation: - self.navigation = FakeNavigation(self.main_frame, url) - self.handlers["request"](self.navigation) - return self.navigation - - def respond(self, status: int, *, request: FakeNavigation | None = None) -> None: - self.handlers["response"](FakeDocument(status, request or self.navigation)) - - def fail(self, request: FakeNavigation, reason: str) -> None: - request.failure = reason - self.handlers["requestfailed"](request) + def on(self, event: str, callback: Callable[[FakeResponse], None]) -> None: + self.handlers[event] = callback - def commit(self, url: str, html: str, *, loaded: bool = True) -> None: - self.url = url - self.html = html - if callback := self.handlers.get("framenavigated"): - callback(self.main_frame) - if loaded: - self.load() - - def load(self) -> None: - self.handlers["domcontentloaded"](self) - - async def goto(self, url: str, **_options: object) -> None: - self.begin_navigation(url) - self.connection.pause("initial", url) - await self.connection.continued["initial"].wait() - if self.navigation_error is not None: - raise self.navigation_error - self.respond(self.context.driver.initial_status) - self.commit(url, "challenge") - self.started.set() + async def goto(self, url: str, *, wait_until: str, **_options: object) -> None: + assert wait_until == "networkidle" + driver = self.context.driver + self.url = driver.destination or url + driver.started.put_nowait(self) + if driver.block_pages: + await self.ready.wait() + if driver.error is not None: + raise driver.error + for status, navigation, main_frame in driver.responses: + frame = self.main_frame if main_frame else object() + self.handlers["response"](FakeResponse(status, FakeRequest(frame, navigation))) async def content(self) -> str: - html = self.html - self.content_started.set() - if self.content_gate is not None: - await self.content_gate.wait() - return html + return "保研通知" async def close(self) -> None: - if not self.closed: - self.close_started.set() - if self.close_gate is not None: - await self.close_gate.wait() - self.closed = True - self.context.driver.active_pages -= 1 + self.closed = True -class FakeRouteRequest: - def __init__(self, page: FakePage, resource_type: str) -> None: - self.frame = FakeFrame(page) - self.resource_type = resource_type +class FakeDownload: + def __init__(self, url: str, body: bytes) -> None: + self.url = url + self.status = 200 + self.headers = {"content-type": "application/pdf"} + self.data = body + self.disposed = False + async def body(self) -> bytes: + return self.data + + async def dispose(self) -> None: + self.disposed = True -class FakeRoute: - def __init__(self, page: FakePage, resource_type: str) -> None: - self.request = FakeRouteRequest(page, resource_type) - self.continued = False - self.aborted = False - async def continue_(self) -> None: - self.continued = True +class FakeDownloads: + def __init__(self, context: FakeContext) -> None: + self.context = context + self.responses: list[FakeDownload] = [] - async def abort(self) -> None: - self.aborted = True + async def get(self, url: str, *, headers: dict[str, str], **_options: object) -> FakeDownload: + if self.context.driver.error is not None: + raise self.context.driver.error + response = FakeDownload( + url, f"%PDF context {self.context.number}: {headers['Referer']}".encode() + ) + self.responses.append(response) + return response class FakeContext: - def __init__(self, driver: FakeBrowser) -> None: + def __init__(self, driver: FakeBrowser, number: int) -> None: self.driver = driver - self.client = httpx.AsyncClient(transport=driver.transport) - self.request = FakeDownloads(self.client) + self.number = number + self.request = FakeDownloads(self) self.pages: list[FakePage] = [] self.closed = False - self.route_handler: Callable[[FakeRoute], Awaitable[None]] | None = None - self.page_gate: asyncio.Event | None = None - async def route(self, pattern: str, handler: Callable[[FakeRoute], Awaitable[None]]) -> None: - self.route_handler = handler + async def route(self, _pattern: str, _handler: object) -> None: + pass - def on(self, event: str, handler: object) -> None: + def on(self, _event: str, _handler: object) -> None: pass async def new_page(self) -> FakePage: page = FakePage(self) self.pages.append(page) - self.driver.active_pages += 1 - self.driver.peak_pages = max(self.driver.peak_pages, self.driver.active_pages) - self.driver.created.put_nowait(page) - if self.page_gate is not None: - await self.page_gate.wait() return page - async def new_cdp_session(self, page: FakePage) -> FakeCDP: - return page.connection - async def close(self) -> None: self.closed = True - for page in self.pages: - await page.close() - await self.client.aclose() class FakeBrowser: - def __init__(self, transport: httpx.AsyncBaseTransport | None = None) -> None: - self.transport = transport or httpx.MockTransport(lambda _: httpx.Response(404)) + def __init__(self) -> None: self.contexts: list[FakeContext] = [] - self.created: asyncio.Queue[FakePage] = asyncio.Queue() - self.active_pages = 0 - self.peak_pages = 0 - self.initial_status = 202 - self.context_created = asyncio.Event() - self.context_gate: asyncio.Event | None = None - - async def new_context( - self, *, user_agent: str, service_workers: str, accept_downloads: bool - ) -> FakeContext: - context = FakeContext(self) + self.started: asyncio.Queue[FakePage] = asyncio.Queue() + self.block_pages = False + self.destination: str | None = None + self.error: PlaywrightError | None = None + self.responses = [(200, True, True)] + + async def new_context(self, **_options: object) -> FakeContext: + context = FakeContext(self, len(self.contexts) + 1) self.contexts.append(context) - self.context_created.set() - if self.context_gate is not None: - await self.context_gate.wait() return context -def use_browser(monkeypatch: pytest.MonkeyPatch, driver: FakeBrowser) -> None: +@pytest.fixture +def driver(monkeypatch: pytest.MonkeyPatch) -> FakeBrowser: + browser = FakeBrowser() + async def start(_owner: object) -> FakeBrowser: - return driver + return browser monkeypatch.setattr("crawler.crawl.browser.Browser._start", start) + return browser -def test_plain_pages_and_unused_browser_sessions_do_not_start_a_browser( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def unexpected_browser(_fetcher: Fetcher) -> None: - pytest.fail("A browser was started without a browser page request") - - monkeypatch.setattr(Fetcher, "browser", unexpected_browser) - - async def main() -> None: - transport = httpx.MockTransport( - lambda request: httpx.Response(200, content=request.url.path) - ) - async with Fetcher(transport) as fetcher: - async with fetcher.session(False) as pages: - listing = await pages.get("https://a/list") - attachment = await pages.download("https://b/file.pdf", page_url=listing.url) - async with fetcher.session(True): - pass - assert listing.body == b"/list" - assert attachment.body == b"/file.pdf" - - run(main) - - -def test_round_reuses_its_browser_session_and_closes_it_before_the_next_round( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = FakeSessions() - - def browser_service(_fetcher: Fetcher) -> FakeSessions: - return service - - monkeypatch.setattr(Fetcher, "browser", browser_service) - - async def main() -> None: - async with Fetcher(httpx.MockTransport(lambda _: httpx.Response(500))) as fetcher: - async with fetcher.session(True) as pages: - assert service.sessions == [] - listing, notice = await asyncio.gather( - pages.get("https://a/list"), pages.get("https://a/notice") - ) - attachment = await pages.download("https://a/file.pdf", page_url=notice.url) - assert listing.body == notice.body == b"session 1" - assert attachment.body == b"session 1: https://a/notice" - assert not service.sessions[0].closed - - with pytest.raises(RuntimeError, match="closed"): - await pages.get("https://a/after-close") - with pytest.raises(RuntimeError, match="closed"): - await pages.download("https://a/file.pdf", page_url=notice.url) - assert service.sessions[0].closed - - async with fetcher.session(True) as next_round: - assert (await next_round.get("https://a/list")).body == b"session 2" - assert service.sessions[0].closed - assert all(session.closed for session in service.sessions) - - run(main) - - -@pytest.mark.parametrize("challenge", [202, 412]) -def test_challenge_waits_for_the_final_document_to_commit_before_reading_html( - monkeypatch: pytest.MonkeyPatch, challenge: int -) -> None: - driver = FakeBrowser() - driver.initial_status = challenge - use_browser(monkeypatch, driver) - - async def main() -> None: - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - await page.started.wait() - await asyncio.sleep(0) - assert not task.done() - - # A response event precedes commit, even when both documents use the same URL. - page.begin_navigation("https://a.test/list") - page.connection.pause("challenge", "https://a.test/list") - await page.connection.next_command("Fetch.continueRequest", request_id="challenge") - page.respond(200) - await asyncio.sleep(0) - await asyncio.sleep(0) - assert not task.done() - - page.commit("https://a.test/list", "script pending", loaded=False) - await asyncio.sleep(0) - await asyncio.sleep(0) - assert not task.done() - page.html = "rendered notice" - page.load() - result = await task - assert result.status == 200 - assert result.body == b"rendered notice" - assert page.closed - assert driver.active_pages == 0 - finally: - await browser.close() - - run(main) - - -def test_page_limit_is_shared_across_rounds_and_cancellation_frees_a_slot( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - finish = asyncio.Event() - - async def render(_request: object) -> Response: - await finish.wait() - return Response("https://a.test/list", 200, "text/html", b"rendered") +def test_page_visits_and_http_requests_wait_for_each_other(driver: FakeBrowser) -> None: + driver.block_pages = True + clock = FakeClock() + http_started = asyncio.Event() + finish_http = asyncio.Event() + request_times: list[float] = [] - monkeypatch.setattr("crawler.crawl.browser._Page.run", render) + async def handle(_request: httpx.Request) -> httpx.Response: + request_times.append(clock.now) + http_started.set() + await finish_http.wait() + return httpx.Response(200) async def main() -> None: - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - try: - first, second = await asyncio.gather(browser.session(), browser.session()) - tasks = [ - asyncio.create_task(first.get("https://a.test/1")), - asyncio.create_task(second.get("https://b.test/2")), - asyncio.create_task(first.get("https://a.test/3")), - ] - page_one = await driver.created.get() - await driver.created.get() - await asyncio.sleep(0) - assert driver.active_pages == driver.peak_pages == 2 - assert driver.created.empty() - - tasks[0].cancel() - with pytest.raises(asyncio.CancelledError): - await tasks[0] - page_three = await driver.created.get() - assert page_one.closed - assert not page_three.closed - assert driver.active_pages == driver.peak_pages == 2 - - finish.set() - assert [response.body for response in await asyncio.gather(*tasks[1:])] == [ - b"rendered", - b"rendered", - ] - assert driver.active_pages == 0 - finally: - finish.set() - await browser.close() - assert all(context.closed for context in driver.contexts) + async with ( + Fetcher(httpx.MockTransport(handle), clock.clock, clock.sleep) as fetcher, + fetcher.session(True, url="https://a/list") as session, + ): + first_http = asyncio.create_task(fetcher.get("https://a/first")) + await http_started.wait() + visit = asyncio.create_task(session.get("https://a/list")) + await asyncio.sleep(0) + assert driver.started.empty() + clock.now = 2 + finish_http.set() + await first_http + page = await driver.started.get() + assert clock.now == 2 + INTERVAL + second_http = asyncio.create_task(fetcher.get("https://a/second")) + await asyncio.sleep(0) + assert request_times == [0] + clock.now = 10 + page.ready.set() + await visit + await second_http + assert page.closed + assert request_times == [0, 10 + INTERVAL] run(main) -def test_navigation_during_html_read_discards_the_previous_document( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - driver.initial_status = 200 - use_browser(monkeypatch, driver) +def test_page_limit_is_shared_by_rounds(driver: FakeBrowser) -> None: + driver.block_pages = True async def main() -> None: - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - page.content_gate = asyncio.Event() - await page.content_started.wait() - previous = page.navigation - page.begin_navigation("https://a.test/final") - page.content_gate.set() - await asyncio.sleep(0) - await asyncio.sleep(0) - assert not task.done() - - page.respond(200, request=previous) - await asyncio.sleep(0) - assert not task.done() - page.connection.pause("navigate", "https://a.test/final") - await page.connection.next_command("Fetch.continueRequest", request_id="navigate") - page.respond(200) - await asyncio.sleep(0) - assert not task.done() - page.commit("https://a.test/final", "final document") - result = await task - assert result.url == "https://a.test/final" - assert result.body == b"final document" - finally: - await browser.close() + async with ( + Fetcher() as fetcher, + fetcher.session(True, url="https://a/list") as first, + fetcher.session(True, url="https://b/list") as second, + ): + tasks = [ + asyncio.create_task(first.get("https://a/list")), + asyncio.create_task(second.get("https://b/list")), + asyncio.create_task(first.get("https://c/list")), + ] + first_page = await driver.started.get() + second_page = await driver.started.get() + await asyncio.sleep(0) + assert sum(len(context.pages) for context in driver.contexts) == 2 + first_page.ready.set() + third_page = await driver.started.get() + assert first_page.closed + second_page.ready.set() + third_page.ready.set() + await asyncio.gather(*tasks) + assert all(page.closed for context in driver.contexts for page in context.pages) run(main) -def test_attachment_cookies_are_scoped_to_the_round_and_each_redirect_destination( - monkeypatch: pytest.MonkeyPatch, -) -> None: - requested: list[tuple[str, str, str]] = [] - - def handle(request: httpx.Request) -> httpx.Response: - requested.append( - ( - str(request.url), - request.headers.get("cookie", ""), - request.headers.get("referer", ""), - ) - ) - if request.url.host == "a.test": - if request.headers.get("cookie") != "challenge=passed": - return httpx.Response(403, content=b"challenge required") - return httpx.Response(302, headers={"location": "https://b.test/file.pdf"}) - assert "challenge" not in request.headers.get("cookie", "") - return httpx.Response(200, headers={"content-type": "application/pdf"}, content=b"PDF") - - driver = FakeBrowser(httpx.MockTransport(handle)) - use_browser(monkeypatch, driver) +def test_each_round_opens_reuses_and_closes_its_context(driver: FakeBrowser) -> None: clock = FakeClock() async def main() -> None: async with Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - context = driver.contexts[0] - context.client.cookies.set("challenge", "passed", domain="a.test", path="/private") - result = await session.download( - "https://a.test/private/file.pdf", page_url="https://a.test/notice" - ) - assert (result.url, result.status, result.body) == ( - "https://b.test/file.pdf", - 200, - b"PDF", - ) - assert all(response.disposed for response in context.request.responses) - outside_cookie_path = await session.download( - "https://a.test/public/file.pdf", page_url="https://a.test/notice" - ) - assert outside_cookie_path.status == 403 - await session.close() - - next_round = await browser.session() - missing_cookie = await next_round.download( - "https://a.test/private/file.pdf", page_url="https://a.test/notice" - ) - assert missing_cookie.status == 403 - assert clock.now >= INTERVAL - finally: - await browser.close() - - assert requested == [ - ("https://a.test/private/file.pdf", "challenge=passed", "https://a.test/notice"), - ("https://b.test/file.pdf", "", "https://a.test/notice"), - ("https://a.test/public/file.pdf", "", "https://a.test/notice"), - ("https://a.test/private/file.pdf", "", "https://a.test/notice"), - ] + async with fetcher.session(False, url="https://a/list"): + assert driver.contexts == [] + async with fetcher.session(True, url="https://a/list") as session: + assert len(driver.contexts) == 1 + await session.get("https://a/list") + await session.get("https://a/notice") + assert len(driver.contexts[0].pages) == 2 + assert not driver.contexts[0].closed + assert driver.contexts[0].closed + async with fetcher.session(True, url="https://a/list"): + assert len(driver.contexts) == 2 + assert driver.contexts[1].closed run(main) -def test_page_permit_covers_challenge_refresh_and_subresources_until_the_page_closes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) +def test_downloads_use_the_rounds_context_request_client(driver: FakeBrowser) -> None: clock = FakeClock() - ordinary_started = asyncio.Event() - permits: list[str] = [] - - def handle(_request: httpx.Request) -> httpx.Response: - ordinary_started.set() - return httpx.Response(200, content=b"next request") async def main() -> None: - async with Fetcher(httpx.MockTransport(handle), clock.clock, clock.sleep) as fetcher: - - async def permit(url: str) -> Permit: - permits.append(url) - return await fetcher.permit(url) - - browser = Browser(permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - await page.started.wait() - configuration = await page.connection.next_command("Fetch.enable") - assert configuration == { - "patterns": [ - {"urlPattern": "*", "requestStage": "Request", "resourceType": "Document"} - ] - } - - route = driver.contexts[0].route_handler - assert route is not None - for resource_type in ("script", "stylesheet", "xhr", "document"): - resource = FakeRoute(page, resource_type) - await route(resource) - assert resource.continued - assert not resource.aborted - image = FakeRoute(page, "image") - await route(image) - assert image.aborted - - page.connection.pause("iframe", "https://b.test/frame", frame_id="child") - await page.connection.next_command("Fetch.continueRequest", request_id="iframe") - page.begin_navigation("https://a.test/list") - page.connection.pause("challenge", "https://a.test/list") - await page.connection.next_command("Fetch.continueRequest", request_id="challenge") - assert permits == ["https://a.test/list"] - assert clock.now == 0 - - ordinary = asyncio.create_task(fetcher.get("https://a.test/other")) - await asyncio.sleep(0) - assert not ordinary_started.is_set() - page.respond(200) - page.commit("https://a.test/list", "ready") - assert (await task).body == b"ready" - assert page.closed - assert (await ordinary).body == b"next request" - assert clock.now == INTERVAL - finally: - await browser.close() + async with Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher: + for number in (1, 2): + async with fetcher.session(True, url="https://a/list") as session: + await session.get("https://a/notice") + attachment = await session.download( + "https://a/file.pdf", page_url="https://a/notice" + ) + assert attachment.body == f"%PDF context {number}: https://a/notice".encode() + assert attachment.content_type == "application/pdf" + assert driver.contexts[number - 1].request.responses[0].disposed run(main) -def test_opposite_cross_site_redirects_release_old_permits_before_waiting( - monkeypatch: pytest.MonkeyPatch, +def test_last_main_navigation_response_is_used_without_a_cross_site_permit( + driver: FakeBrowser, ) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - clock = FakeClock() + driver.destination = "https://b/notice" + driver.responses = [ + (202, True, True), + (200, True, True), + (404, False, True), + (500, True, False), + ] async def main() -> None: - async with Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher: - browser = Browser(fetcher.permit) + async with Fetcher() as fetcher: + destination_permit = await fetcher.permit("https://b/busy") try: - session = await browser.session() - first = asyncio.create_task(session.get("https://a.test/list")) - second = asyncio.create_task(session.get("https://b.test/list")) - page_a = await driver.created.get() - page_b = await driver.created.get() - await asyncio.gather(page_a.started.wait(), page_b.started.wait()) - page_a.begin_navigation("https://b.test/final") - page_b.begin_navigation("https://a.test/final") - page_a.connection.pause("to-b", "https://b.test/final", redirected="initial") - page_b.connection.pause("to-a", "https://a.test/final", redirected="initial") - await asyncio.gather( - page_a.connection.next_command("Fetch.continueRequest", request_id="to-b"), - page_b.connection.next_command("Fetch.continueRequest", request_id="to-a"), - ) - - page_a.respond(200) - page_a.commit("https://b.test/final", "from b") - page_b.respond(200) - page_b.commit("https://a.test/final", "from a") - responses = await asyncio.gather(first, second) - assert [(response.url, response.body) for response in responses] == [ - ("https://b.test/final", b"from b"), - ("https://a.test/final", b"from a"), - ] - assert page_a.closed - assert page_b.closed - assert clock.now >= INTERVAL + async with fetcher.session(True, url="https://a/list") as session: + response = await session.get("https://a/notice") + assert response.status == 200 + assert response.url == "https://b/notice" + assert response.body.decode() == "保研通知" finally: - await browser.close() + destination_permit.release() run(main) -def test_same_document_url_change_during_html_read_keeps_document_readiness( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - driver.initial_status = 200 - use_browser(monkeypatch, driver) +@pytest.mark.parametrize("status", [202, 412]) +def test_unresolved_challenges_fail(driver: FakeBrowser, status: int) -> None: + driver.responses = [(status, True, True)] async def main() -> None: - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - page.content_gate = asyncio.Event() - await page.content_started.wait() - page.commit("https://a.test/list?view=all#notices", page.html, loaded=False) - page.content_gate.set() - result = await task - assert result.url == "https://a.test/list?view=all#notices" - assert result.body == b"challenge" - assert page.closed - finally: - await browser.close() + async with Fetcher() as fetcher, fetcher.session(True, url="https://a/list") as session: + with pytest.raises(FetchError) as caught: + await session.get("https://a/list") + assert caught.value.failure == "unreachable" + assert driver.contexts[0].pages[0].closed run(main) @@ -736,448 +309,68 @@ async def main() -> None: @pytest.mark.parametrize( ("error", "failure"), [ - (None, None), - (PlaywrightTimeoutError("Page.goto timed out"), "timeout"), - (PlaywrightError("net::ERR_CONNECTION_RESET"), "unreachable"), - (PlaywrightError("Page.goto: Download is starting"), "unreachable"), - (PlaywrightError("Unexpected browser error"), None), + (PlaywrightTimeoutError("Timed out"), "timeout"), + (PlaywrightError("Browser request failed"), "unreachable"), ], - ids=["cancel", "timeout", "unreachable", "unexpected_download", "unexpected_error"], ) -def test_navigation_failure_or_cancellation_closes_the_page_and_releases_its_permit( - monkeypatch: pytest.MonkeyPatch, - error: PlaywrightError | None, - failure: Failure | None, +@pytest.mark.parametrize("download", [False, True]) +def test_playwright_errors_become_fetch_errors( + driver: FakeBrowser, error: PlaywrightError, failure: Failure, *, download: bool ) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) + driver.error = error clock = FakeClock() async def main() -> None: - transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"still usable")) - async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - if error is None: - await page.started.wait() - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - else: - page.navigation_error = error - if failure is None: - with pytest.raises(PlaywrightError) as browser_error: - await task - assert browser_error.value is error - else: - with pytest.raises(FetchError) as source_failure: - await task - assert source_failure.value.failure == failure - assert page.closed - assert driver.active_pages == 0 - assert (await fetcher.get("https://a.test/after-failure")).body == b"still usable" - assert clock.now == INTERVAL - finally: - await browser.close() - - run(main) - - -def test_cancelling_a_cross_site_wait_leaves_both_sites_usable( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - clock = FakeClock() - destination_requested = asyncio.Event() - - async def main() -> None: - transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"available")) - async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: - - async def permit(url: str) -> Permit: - if url == "https://b.test/final": - destination_requested.set() - return await fetcher.permit(url) - - browser = Browser(permit) - destination = await fetcher.permit("https://b.test/held") - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - await page.started.wait() - page.begin_navigation("https://b.test/final") - page.connection.pause("redirect", "https://b.test/final", redirected="initial") - await destination_requested.wait() - assert (await fetcher.get("https://a.test/other")).body == b"available" - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - assert page.closed - destination.release() - assert (await fetcher.get("https://b.test/other")).body == b"available" - finally: - destination.release() - await browser.close() - - run(main) - - -def test_same_site_http_redirect_limit_does_not_count_challenge_refreshes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - clock = FakeClock() - - async def main() -> None: - transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"available")) - async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - await page.started.wait() - for index in range(MAX_REDIRECTS + 2): - request_id = f"challenge-{index}" - page.begin_navigation("https://a.test/list") - page.connection.pause(request_id, "https://a.test/list") - await page.connection.next_command( - "Fetch.continueRequest", request_id=request_id - ) - previous = "initial" - for index in range(MAX_REDIRECTS): - request_id = f"redirect-{index}" - page.begin_navigation(f"https://a.test/redirect/{index}") - page.connection.pause( - request_id, f"https://a.test/redirect/{index}", redirected=previous - ) - await page.connection.next_command( - "Fetch.continueRequest", request_id=request_id - ) - previous = request_id - page.begin_navigation("https://a.test/too-far") - page.connection.pause("over-limit", "https://a.test/too-far", redirected=previous) - with pytest.raises(FetchError) as caught: - await task - assert caught.value.failure == "too_many_redirects" - assert not page.connection.continued["over-limit"].is_set() - assert page.closed - assert (await fetcher.get("https://a.test/other")).body == b"available" - assert clock.now == INTERVAL - finally: - await browser.close() - - run(main) - - -def test_challenge_navigation_network_failure_releases_the_page_without_waiting_for_deadline( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - clock = FakeClock() - - async def main() -> None: - transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"available")) - async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - await page.started.wait() - page.begin_navigation("https://a.test/after-challenge") - page.connection.pause("challenge", "https://a.test/after-challenge") - await page.connection.next_command("Fetch.continueRequest", request_id="challenge") - page.fail(page.navigation, "net::ERR_CONNECTION_REFUSED") - - with pytest.raises(FetchError) as caught: - await task - assert caught.value.failure == "unreachable" - assert caught.value.url == "https://a.test/after-challenge" - assert page.closed - assert (await fetcher.get("https://a.test/other")).body == b"available" - assert clock.now == INTERVAL - finally: - await browser.close() - - run(main) - - -def test_aborted_navigation_and_old_or_subresource_failures_allow_the_replacement_document( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - - async def main() -> None: - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - await page.started.wait() - previous = page.navigation - page.fail(previous, "net::ERR_ABORTED") - await asyncio.sleep(0) - assert not task.done() - - page.begin_navigation("https://a.test/final") - page.fail(previous, "net::ERR_CONNECTION_REFUSED") - script = FakeNavigation( - page.main_frame, "https://a.test/optional.js", navigation=False - ) - page.fail(script, "net::ERR_CONNECTION_REFUSED") - iframe = FakeNavigation(FakeFrame(page), "https://b.test/optional-frame") - page.fail(iframe, "net::ERR_CONNECTION_REFUSED") - await asyncio.sleep(0) - assert not task.done() - - page.connection.pause("replacement", "https://a.test/final") - await page.connection.next_command( - "Fetch.continueRequest", request_id="replacement" - ) - page.respond(200) - page.commit("https://a.test/final", "replacement document") - assert (await task).body == b"replacement document" - assert page.closed - finally: - await browser.close() - - run(main) - - -def test_cancelling_context_creation_waits_for_and_closes_the_unclaimed_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - driver = FakeBrowser() - driver.context_gate = asyncio.Event() - use_browser(monkeypatch, driver) - - async def main() -> None: - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - task = asyncio.create_task(browser.session()) - try: - await driver.context_created.wait() - task.cancel() - await asyncio.sleep(0) - assert not task.done() - assert not driver.contexts[0].closed - assert driver.context_gate is not None - driver.context_gate.set() - with pytest.raises(asyncio.CancelledError): - await task - assert driver.contexts[0].closed - assert not browser.sessions - finally: - assert driver.context_gate is not None - driver.context_gate.set() - await asyncio.gather(task, return_exceptions=True) - await browser.close() - for context in driver.contexts: - await context.close() - - run(main) - - -@pytest.mark.parametrize("expired", [False, True], ids=["cancel", "timeout"]) -def test_interrupted_page_creation_closes_the_page_before_releasing_its_slot( - monkeypatch: pytest.MonkeyPatch, expired: bool -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - - async def main() -> None: - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - session = await browser.session() - context = driver.contexts[0] - context.page_gate = asyncio.Event() - deadlines: list[asyncio.Timeout] = [] - timeout = asyncio.timeout - - def deadline(_seconds: float) -> asyncio.Timeout: - timer = timeout(None) - deadlines.append(timer) - return timer - - monkeypatch.setattr("crawler.crawl.browser.asyncio.timeout", deadline) - task = asyncio.create_task(session.get("https://a.test/list")) - page = await driver.created.get() - page.close_gate = asyncio.Event() - blocked: asyncio.Task[bool] | None = None - try: - if expired: - deadlines[0].reschedule(asyncio.get_running_loop().time()) - else: - task.cancel() - await asyncio.sleep(0) - await asyncio.sleep(0) - assert not task.done() - context.page_gate.set() - await page.close_started.wait() - await browser.slots.acquire() - blocked = asyncio.create_task(browser.slots.acquire()) - await asyncio.sleep(0) - assert not blocked.done() - assert not page.closed - page.close_gate.set() - if expired: - with pytest.raises(FetchError) as caught: - await task - assert caught.value.failure == "timeout" - else: - with pytest.raises(asyncio.CancelledError): - await task - await blocked - browser.slots.release() - browser.slots.release() - assert page.closed - assert driver.active_pages == 0 - finally: - context.page_gate.set() - page.close_gate.set() - if blocked is not None and not blocked.done(): - blocked.cancel() - await asyncio.gather(task, return_exceptions=True) - if blocked is not None: - await asyncio.gather(blocked, return_exceptions=True) - await browser.close() - - run(main) - - -@pytest.mark.parametrize("page_creation", [False, True], ids=["context", "page"]) -def test_creation_failure_after_cancellation_is_propagated_once( - monkeypatch: pytest.MonkeyPatch, page_creation: bool -) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - - async def main() -> None: - started, finish = asyncio.Event(), asyncio.Event() - error = RuntimeError("Resource creation failed") - unhandled: list[dict[str, object]] = [] - loop = asyncio.get_running_loop() - previous_handler = loop.get_exception_handler() - - def unexpected(_loop: asyncio.AbstractEventLoop, context: dict[str, object]) -> None: - unhandled.append(context) - - loop.set_exception_handler(unexpected) - async with Fetcher() as fetcher: - browser = Browser(fetcher.permit) - if page_creation: - session = await browser.session() - - async def create_page() -> FakePage: - started.set() - await finish.wait() - raise error - - monkeypatch.setattr(driver.contexts[0], "new_page", create_page) - - async def operation() -> object: - return await session.get("https://a.test/list") - - else: - - async def create_context(**_options: object) -> FakeContext: - started.set() - await finish.wait() - raise error - - monkeypatch.setattr(driver, "new_context", create_context) - - async def operation() -> object: - return await browser.session() - - task = asyncio.create_task(operation()) - try: - await started.wait() - task.cancel() - await asyncio.sleep(0) - assert not task.done() - finish.set() - with pytest.raises(RuntimeError) as caught: - await task - assert caught.value is error - await asyncio.sleep(0) - assert not unhandled - assert driver.active_pages == 0 - finally: - finish.set() - await asyncio.gather(task, return_exceptions=True) - loop.set_exception_handler(previous_handler) - await browser.close() + async with ( + Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher, + fetcher.session(True, url="https://a/list") as session, + ): + operation = ( + session.download("https://a/file.pdf", page_url="https://a/notice") + if download + else session.get("https://a/notice") + ) + with pytest.raises(FetchError) as caught: + await operation + assert caught.value.failure == failure + driver.error = None + assert (await session.get("https://a/retry")).status == 200 + assert clock.now == INTERVAL run(main) +@pytest.mark.parametrize("stage", ["start", "context", "configure"]) @pytest.mark.parametrize( - ("message", "failure"), + ("error", "failure"), [ - ("APIRequestContext.get: self-signed certificate", "unreachable"), - ("APIRequestContext.get: certificate has expired", "unreachable"), - ("APIRequestContext.get: unable to verify the first certificate", "unreachable"), - ( - "APIRequestContext.get: Hostname/IP does not match certificate's altnames:" - " Host: a.test", - "unreachable", - ), - ( - "APIRequestContext.get: failed to decompress 'gzip' encoding: incorrect header", - "unreachable", - ), - ("APIRequestContext.get: Unknown internal failure", None), - ( - "APIRequestContext.get: Unknown internal failure\nCall log:\n" - " - User-Agent: ECONNRESET self-signed certificate\n", - None, - ), + (PlaywrightTimeoutError("Timed out"), "timeout"), + (PlaywrightError("Browser initialization failed"), "unreachable"), ], - ids=["self_signed", "expired", "unverified", "hostname", "compression", "unknown", "call_log"], ) -def test_attachment_transport_errors_release_the_permit_and_preserve_unknown_errors( - monkeypatch: pytest.MonkeyPatch, message: str, failure: Failure | None +def test_initialization_errors_include_the_source_url( + monkeypatch: pytest.MonkeyPatch, + driver: FakeBrowser, + stage: str, + error: PlaywrightError, + failure: Failure, ) -> None: - driver = FakeBrowser() - use_browser(monkeypatch, driver) - clock = FakeClock() + async def fail(*_args: object, **_options: object) -> None: + raise error + + if stage == "start": + monkeypatch.setattr(Browser, "_start", fail) + elif stage == "context": + monkeypatch.setattr(FakeBrowser, "new_context", fail) + else: + monkeypatch.setattr(FakeContext, "route", fail) async def main() -> None: - transport = httpx.MockTransport(lambda _: httpx.Response(200, content=b"still available")) - async with Fetcher(transport, clock.clock, clock.sleep) as fetcher: - browser = Browser(fetcher.permit) - try: - session = await browser.session() - error = PlaywrightError(message) - driver.contexts[0].request.error = error - if failure is None: - with pytest.raises(PlaywrightError) as caught: - await session.download( - "https://a.test/file.pdf", page_url="https://a.test/notice" - ) - assert caught.value is error - else: - with pytest.raises(FetchError) as source_failure: - await session.download( - "https://a.test/file.pdf", page_url="https://a.test/notice" - ) - assert source_failure.value.failure == failure - assert ( - await fetcher.get("https://a.test/after-failure") - ).body == b"still available" - assert clock.now == INTERVAL - finally: - await browser.close() + async with Fetcher() as fetcher: + with pytest.raises(FetchError) as caught: + await fetcher.browser().session("https://a/list") + assert caught.value.failure == failure + assert caught.value.url == "https://a/list" + assert all(context.closed for context in driver.contexts) run(main) diff --git a/crawler/src/crawler/crawl/contract.py b/crawler/src/crawler/crawl/contract.py index e3cce86..925d454 100644 --- a/crawler/src/crawler/crawl/contract.py +++ b/crawler/src/crawler/crawl/contract.py @@ -29,7 +29,7 @@ class PendingNotice(BaseModel): step: Annotated[ Literal["screen", "fetch"], Field( - description="screen：还没初筛；fetch：初筛通过但最新快照还没抽取的，或者项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的" + description="screen：还没初筛；fetch：要抓正文，初筛通过还没抽取的，或者项目还没截止、一天没抓过的" ), ] @@ -114,14 +114,6 @@ class AttachmentInfo(BaseModel): url: Annotated[str, Field(max_length=2000)] content_type: Annotated[str, Field(max_length=200, min_length=1)] text: Annotated[str, Field(max_length=2097152)] - error: Annotated[ - str | None, - Field( - description="附件下载失败原因；不写表示下载正常，不影响通知正文和其他附件", - max_length=2000, - min_length=1, - ), - ] = None type ValueItem = Annotated[str, Field(max_length=500, min_length=1)] diff --git a/crawler/src/crawler/crawl/fetch.py b/crawler/src/crawler/crawl/fetch.py index 9897916..540769a 100644 --- a/crawler/src/crawler/crawl/fetch.py +++ b/crawler/src/crawler/crawl/fetch.py @@ -42,21 +42,14 @@ class _Site: finished_at: float = float("-inf") -def site_key(url: str) -> str: - return httpx.URL(url).netloc.decode("ascii") - - @dataclass class Permit: _site: _Site _clock: Callable[[], float] - _released: bool = False def release(self) -> None: - if not self._released: - self._released = True - self._site.finished_at = self._clock() - self._site.lock.release() + self._site.finished_at = self._clock() + self._site.lock.release() class Fetcher: @@ -86,8 +79,8 @@ async def __aexit__(self, *_: object) -> None: finally: await self._client.aclose() - def session(self, browser: bool) -> PageSession: - return PageSession(self, browser) + def session(self, browser: bool, *, url: str) -> PageSession: + return PageSession(self, browser, url) def browser(self) -> Browser: if self._browser is None: @@ -98,7 +91,7 @@ def browser(self) -> Browser: return self._browser async def permit(self, url: str) -> Permit: - site = self._sites.setdefault(site_key(url), _Site()) + site = self._sites.setdefault(httpx.URL(url).netloc.decode("ascii"), _Site()) await site.lock.acquire() try: wait = site.finished_at + INTERVAL - self._clock() @@ -136,37 +129,27 @@ async def _send(self, url: str) -> httpx.Response: class PageSession: - def __init__(self, fetcher: Fetcher, browser: bool) -> None: + def __init__(self, fetcher: Fetcher, browser: bool, url: str) -> None: self._fetcher = fetcher self._use_browser = browser + self._url = url self._browser_session: BrowserSession | None = None - self._lock = asyncio.Lock() - self._closed = False async def __aenter__(self) -> Self: + if self._use_browser: + self._browser_session = await self._fetcher.browser().session(self._url) return self async def __aexit__(self, *_: object) -> None: - self._closed = True if self._browser_session is not None: await self._browser_session.close() - async def _open_browser(self) -> BrowserSession: - async with self._lock: - if self._browser_session is None: - self._browser_session = await self._fetcher.browser().session() - return self._browser_session - async def get(self, url: str) -> Response: - if self._closed: - raise RuntimeError("Page session is closed") - if self._use_browser: - return await (await self._open_browser()).get(url) + if self._browser_session is not None: + return await self._browser_session.get(url) return await self._fetcher.get(url) async def download(self, url: str, *, page_url: str) -> Response: - if self._closed: - raise RuntimeError("Page session is closed") - if self._use_browser: - return await (await self._open_browser()).download(url, page_url=page_url) + if self._browser_session is not None: + return await self._browser_session.download(url, page_url=page_url) return await self._fetcher.get(url) diff --git a/crawler/src/crawler/crawl/fetch_test.py b/crawler/src/crawler/crawl/fetch_test.py index 4b2f516..2522dba 100644 --- a/crawler/src/crawler/crawl/fetch_test.py +++ b/crawler/src/crawler/crawl/fetch_test.py @@ -88,24 +88,6 @@ def test_a_redirect_to_another_site_waits_its_turn_there() -> None: assert all(gap >= INTERVAL for gap in gaps(sites.times("b"))) -def test_http_and_browser_permissions_share_the_completion_interval() -> None: - sites = FakeSites({"http://a/notice": httpx.Response(200)}) - - async def use(fetcher: Fetcher) -> None: - browser_request = await fetcher.permit("http://A:80/list.htm") - http_request = asyncio.create_task(fetcher.get("http://a/notice")) - await asyncio.sleep(0) - assert not sites.requests - sites.now = 3.0 - browser_request.release() - # A late terminal event for the old request must not release another one's permit. - browser_request.release() - await http_request - - sites.run(use) - assert sites.times("a") == [3.0 + INTERVAL] - - def test_cancelling_the_interval_wait_does_not_mark_a_request_finished() -> None: async def run() -> None: sites = FakeSites({}) diff --git a/crawler/src/crawler/crawl/rounds.py b/crawler/src/crawler/crawl/rounds.py index beef517..334bf21 100644 --- a/crawler/src/crawler/crawl/rounds.py +++ b/crawler/src/crawler/crawl/rounds.py @@ -5,10 +5,11 @@ import logging import time from collections.abc import Awaitable, Callable +from contextlib import AsyncExitStack from dataclasses import dataclass -from crawler.crawl.attachments import Attachment from crawler.crawl.contract import ( + AttachmentInfo, ExtractedProgram, Extraction, ExtractionContext, @@ -39,6 +40,12 @@ def __init__(self, failure: Failure, detail: str | None = None) -> None: self.detail = detail +@dataclass(frozen=True) +class Attachment: + info: AttachmentInfo + data: bytes + + @dataclass(frozen=True) class NoticePage: html: str @@ -78,9 +85,10 @@ async def crawl_source( steps: Steps, clock: Callable[[], float] = time.time, ) -> None: - async with fetcher.session(source.browser) as pages: - started_at = int(clock()) + started_at = int(clock()) + async with AsyncExitStack() as stack: try: + pages = await stack.enter_async_context(fetcher.session(source.browser, url=source.url)) items = await steps.list_items(pages, source) if not items: raise StepError("no_items") @@ -161,11 +169,7 @@ async def _fetch_notice( ), ) for position in result.missing_files: - attachment = page.attachments[position] - if attachment.data is not None: - await website.attachment( - result.snapshot_id, position, attachment.data, url=attachment.info.url - ) + await website.attachment(result.snapshot_id, position, page.attachments[position].data) if result.extraction is None: return 0 programs, usage = await steps.extract(page, result.extraction) diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index 620976b..a54b3fe 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -1,14 +1,11 @@ import asyncio import json import re -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager from typing import Any import httpx import pytest -from crawler.crawl.attachments import Attachment, download_attachment from crawler.crawl.contract import ( AttachmentInfo, ExtractedFields, @@ -20,8 +17,8 @@ Usage, WorkSource, ) -from crawler.crawl.fetch import Fetcher, FetchError, PageSession, Response -from crawler.crawl.rounds import NoticePage, StepError, Steps, crawl_source +from crawler.crawl.fetch import Fetcher, FetchError, PageSession +from crawler.crawl.rounds import Attachment, NoticePage, StepError, Steps, crawl_source from crawler.crawl.website import Website SOURCE = WorkSource( @@ -45,14 +42,11 @@ def __init__(self, pending: list[dict[str, Any]], extracted: frozenset[int]) -> self.pending = pending self.extracted = extracted self.calls: list[tuple[str, Any]] = [] - self.upload_urls: list[str] = [] def handle(self, request: httpx.Request) -> httpx.Response: path = request.url.path.removeprefix("/api/v1/crawler") sent_json = request.headers.get("content-type") == "application/json" body: Any = json.loads(request.content) if sent_json else request.content - if request.method == "PUT": - self.upload_urls.append(request.url.params["url"]) self.calls.append((f"{request.method} {path}", body)) if path.endswith("/listing"): return httpx.Response(200, json={"new_count": 2, "pending": self.pending}) @@ -70,19 +64,14 @@ def handle(self, request: httpx.Request) -> httpx.Response: ) return httpx.Response(204) - def run( - self, - steps: Steps, - source: WorkSource = SOURCE, - transport: httpx.AsyncBaseTransport | None = None, - ) -> None: + def run(self, steps: Steps) -> None: async def main() -> None: - nowhere = transport or httpx.MockTransport(lambda _: httpx.Response(500)) + nowhere = httpx.MockTransport(lambda _: httpx.Response(500)) async with ( Fetcher(nowhere) as fetcher, Website("http://site", "token", httpx.MockTransport(self.handle)) as website, ): - await crawl_source(source, fetcher, website, steps, clock=lambda: 1_800_000_000) + await crawl_source(SOURCE, fetcher, website, steps, clock=lambda: 1_800_000_000) asyncio.run(main()) @@ -176,48 +165,6 @@ async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage } -def test_attachment_errors_are_reported_without_uploading_error_pages_or_losing_the_notice() -> ( - None -): - site = FakeWebsite([pending(1, "fetch")], extracted=frozenset()) - files = { - "https://captcha.test/download": "
请输入验证码下载附件
".encode(), - "https://deleted.test/download": "
获取文件错误
".encode(), - "https://files.test/download?file=form.pdf&owner=2": b"%PDF-1.7\n", - } - - def handle(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - content=files[str(request.url)], - headers={"content-type": "text/html;charset=utf-8"}, - ) - - async def read(pages: PageSession, _source: WorkSource, url: str) -> NoticePage: - attachments = [ - await download_attachment(pages, link, name=f"申请表 {n}", page_url=url) - for n, link in enumerate(files) - ] - return NoticePage("

通知

", "通知", attachments) - - site.run(Steps(list_items, screen, read, extract), transport=httpx.MockTransport(handle)) - - [snapshot] = site.called("POST /notices/1/snapshot") - assert [a["url"] for a in snapshot["attachments"]] == list(files) - assert [a.get("error") for a in snapshot["attachments"]] == [ - "下载附件需要验证码", - "获取文件错误", - None, - ] - assert [a["text"] for a in snapshot["attachments"]] == ["", "", ""] - uploads = [(call, body) for call, body in site.calls if call.startswith("PUT ")] - assert uploads == [("PUT /snapshots/10/attachments/2", b"%PDF-1.7\n")] - assert site.upload_urls == ["https://files.test/download?file=form.pdf&owner=2"] - assert len(site.called("POST /snapshots/10/extraction")) == 1 - [run] = site.called("POST /runs") - assert run["notice_failures"] == [] - - def test_notices_that_fail_are_reported_and_the_round_goes_on() -> None: site = FakeWebsite([pending(n, "fetch") for n in range(1, 5)], extracted=frozenset()) @@ -265,84 +212,20 @@ async def unused(*_: object) -> Any: assert run["listed"] == 0 -@pytest.fixture -def sessions( - monkeypatch: pytest.MonkeyPatch, -) -> tuple[list[tuple[bool, PageSession]], list[PageSession]]: - entered: list[tuple[bool, PageSession]] = [] - exited: list[PageSession] = [] - original_session = Fetcher.session - - @asynccontextmanager - async def session(fetcher: Fetcher, browser: bool) -> AsyncGenerator[PageSession]: - async with original_session(fetcher, browser) as pages: - entered.append((browser, pages)) - try: - yield pages - finally: - exited.append(pages) - - monkeypatch.setattr(Fetcher, "session", session) - return entered, exited - - -def test_a_round_shares_one_session_and_reports_the_rendered_notice( - monkeypatch: pytest.MonkeyPatch, - sessions: tuple[list[tuple[bool, PageSession]], list[PageSession]], -) -> None: - entered, exited = sessions - source = SOURCE.model_copy(update={"browser": True}) - site = FakeWebsite([pending(1, "fetch")], extracted=frozenset({1})) - rendered = "

合成的渲染正文

" - seen: list[PageSession] = [] - - async def get(pages: PageSession, url: str) -> Response: - assert not exited - seen.append(pages) - html = "
  • 合成的列表
" if url == source.url else rendered - return Response(url, 200, "text/html; charset=utf-8", html.encode()) - - async def listing(pages: PageSession, source: WorkSource) -> list[ListedItem]: - response = await pages.get(source.url) - assert "合成的列表" in response.body.decode() - return ITEMS - - async def read(pages: PageSession, _source: WorkSource, url: str) -> NoticePage: - response = await pages.get(url) - return NoticePage(html=response.body.decode(), text="合成的渲染正文", attachments=[]) - - monkeypatch.setattr(PageSession, "get", get) - site.run(Steps(listing, screen, read, extract), source) - - assert len(seen) == 2 - assert seen[0] is seen[1] - assert entered == [(True, seen[0])] - assert exited == [seen[0]] - [snapshot] = site.called("POST /notices/1/snapshot") - assert snapshot["html"] == rendered - assert snapshot["text"] == "合成的渲染正文" - - -@pytest.mark.parametrize("failure", [ValueError, asyncio.CancelledError]) -def test_a_session_exits_when_a_step_fails_or_is_cancelled( - sessions: tuple[list[tuple[bool, PageSession]], list[PageSession]], - failure: type[BaseException], -) -> None: - entered, exited = sessions +def test_a_session_that_cant_start_is_reported(monkeypatch: pytest.MonkeyPatch) -> None: site = FakeWebsite([], extracted=frozenset()) - async def interrupted(pages: PageSession, _source: WorkSource) -> list[ListedItem]: - assert pages is entered[0][1] - raise failure("interrupted") + async def unavailable(_pages: PageSession) -> PageSession: + raise FetchError("unreachable", SOURCE.url) - async def read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: - raise AssertionError + async def unused_read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: + raise AssertionError("Listing failed before notices could be read") - with pytest.raises(failure, match="interrupted"): - site.run(Steps(interrupted, screen, read, extract)) + monkeypatch.setattr(PageSession, "__aenter__", unavailable) + site.run(Steps(list_items, screen, unused_read, extract)) - assert len(entered) == 1 - assert exited == [entered[0][1]] - assert not site.calls - with pytest.raises(RuntimeError, match="Page session is closed"): - asyncio.run(entered[0][1].get(SOURCE.url)) + assert [call for call, _ in site.calls] == ["POST /runs"] + [run] = site.called("POST /runs") + assert run["failure"] == "unreachable" + assert run["detail"] == SOURCE.url + assert run["listed"] == 0 diff --git a/crawler/src/crawler/crawl/website.py b/crawler/src/crawler/crawl/website.py index 2cfbe41..2df1a7e 100644 --- a/crawler/src/crawler/crawl/website.py +++ b/crawler/src/crawler/crawl/website.py @@ -52,10 +52,9 @@ async def snapshot(self, notice_id: int, snapshot: Snapshot) -> SnapshotResult: body = await self._call("POST", f"/notices/{notice_id}/snapshot", snapshot) return SnapshotResult.model_validate_json(body) - async def attachment(self, snapshot_id: int, position: int, data: bytes, *, url: str) -> None: + async def attachment(self, snapshot_id: int, position: int, data: bytes) -> None: response = await self._client.put( f"/snapshots/{snapshot_id}/attachments/{position}", - params={"url": url}, content=data, headers={"Content-Type": "application/octet-stream"}, ) diff --git a/docs/api.md b/docs/api.md index 57e1728..e78d8bb 100644 --- a/docs/api.md +++ b/docs/api.md @@ -101,15 +101,13 @@ | 接口 | 说明 | | --- | --- | | `GET /crawler/work` | 领活：到期要抓的来源和配置，管理员要试抓的地址。发出去的活带租约，到期前不再发；抓取节点中途断了，租约过期后重新发 | -| `POST /crawler/sources/{id}/listing` | 交来源列表页上的条目（网址、标题、日期）。返回这个来源要接着处理的通知：没初筛的；要抓正文的，也就是初筛通过但最新快照还没抽取的，以及项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的（[总体设计 §7.4](design.md#74-发现变化)） | +| `POST /crawler/sources/{id}/listing` | 交来源列表页上的条目（网址、标题、日期）。返回这个来源要接着处理的通知：没初筛的；要抓正文的，也就是初筛通过、还没抽取的，以及项目还没截止、一天没抓过的（[总体设计 §7.4](design.md#74-发现变化)） | | `POST /crawler/sources/{id}/screening` | 交一批通知的初筛分数和是否通过，带模型用量 | -| `POST /crawler/notices/{id}/snapshot` | 交抓到的正文、附件的名称、地址和文字，附件下载失败时附上原因。文字和上一份快照相同就不存新的，下载错误等元数据仍更新。返回快照的编号和还缺文件的附件；要抽取时（有了新快照，或者上一份还没抽取），再附上抽取要对照的东西：通知的发布日期、这所学校在院校清单里的学院、这所学校发布那年到后两年这几届的项目（编号、学院、届、类型、名称、报名时间） | -| `PUT /crawler/snapshots/{id}/attachments/{position}` | 上传一个附件的文件，请求体就是文件本身，查询参数带原下载链接；链接已更新时拒绝旧文件上传 | +| `POST /crawler/notices/{id}/snapshot` | 交抓到的正文、附件的名称、地址和文字。文字和上一份快照相同就不存新的。返回快照的编号和还缺文件的附件；要抽取时（有了新快照，或者上一份还没抽取），再附上抽取要对照的东西：通知的发布日期、这所学校在院校清单里的学院、这所学校发布那年到后两年这几届的项目（编号、学院、届、类型、名称、报名时间） | +| `PUT /crawler/snapshots/{id}/attachments/{position}` | 上传一个附件的文件，请求体就是文件本身 | | `POST /crawler/snapshots/{id}/extraction` | 交抽取的结果，带模型用量。结果是通知涉及的项目，每个带编号（新建的没有）、学院、届、类型、名称、作用和字段，格式见[总体设计 §8.2](design.md#82-抽取)；网站按 §8.3、§8.4 校验和归并 | | `POST /crawler/notices/{id}/removed` | 通知打不开（404、410），标成原文已删除；之后又能打开的，交新快照时取消 | | `POST /crawler/runs` | 交一个来源这一轮的结果：开始和结束时间、成功还是失败和原因、列表里几条、新通知几条、其中有用的几条、哪些通知没抓成和原因。交了才算这一轮结束，失败了也等下一轮再抓 | | `POST /crawler/probes/{id}` | 交试抓的结果：抓到的条目和生成的配置，或者失败的原因，带模型用量 | 一个来源的一轮：领活 → 抓列表页、交列表 → 对没初筛的初筛、交分数 → 对要抓正文的抓正文和附件、交快照、上传附件 → 要抽取的抽取、交结果 → 交这一轮的结果，没抓成的通知和原因写在里面。重抓正文也在来源的这一轮里，守同样的频率，失败了一样记进抓取记录。 - -单个附件下载失败仍保留名称、原下载链接和顺序，附件文字留空，原因写入 `error`：验证码明确标注需要验证码，接口的短错误提示照原文保存，取不到提示时记录 HTTP 状态或传输失败原因。网站不要求上传这些附件；最新快照仍有下载错误或缺文件的通知，即使抽取没有项目，也按一天一次的间隔在来源后续轮次里重抓，下载成功后清除错误并补传文件。同位置、同链接已有的文字和原件在下载失败时保留；仅下载错误变化不创建新快照、不重复抽取。文字不变而下载链接变化时刷新元数据，清除原链接的文件，等待补传。 diff --git a/docs/data-model.md b/docs/data-model.md index c259e1a..2ba1c2d 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -36,7 +36,7 @@ | `crawl_runs` | `ingest` | 一个来源每一轮的结果，抓取看板用：`failure` 为空就是列表读到了，`detail` 写具体原因；各项计数；`notice_failures` 是没抓成的通知和原因 | | `notices` | `ingest` | 列表里的一条通知，`(source_id, url)` 唯一。`screening_score` 和 `screened_in` 是初筛的分数和结论，为空就是还没初筛；`checked_at` 是上次抓正文的时间，重抓用；`removed_at` 有值是原文已删除。抽取出项目的才有版本，没有版本的就是无关 | | `snapshots` | `ingest` | 某次抓到的正文，文字和上一份相同就不存新的；`extracted_at` 为空说明还没交抽取的结果 | -| `attachments` | `ingest` | 快照里的附件：名称、地址、类型、提取出的文字和文件本身。`position` 是在页面上的顺序，证据里的 `attachment` 指的就是它；文件在抓取节点上传后才有。`error` 记录下载失败原因，成功时为空，和原文文字分开保存 | +| `attachments` | `ingest` | 快照里的附件：名称、地址、类型、提取出的文字和文件本身。`position` 是在页面上的顺序，证据里的 `attachment` 指的就是它；文件在抓取节点上传后才有 | | `model_calls` | `ingest` | 判别模型和 LLM 的用量，随产出的结果一起交来 | ## 项目 diff --git a/docs/design.md b/docs/design.md index 0a495e4..1bb2df8 100644 --- a/docs/design.md +++ b/docs/design.md @@ -160,7 +160,7 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 ### 7.3 频率和礼貌 - 平时每个来源每天抓 2 次；3–9 月和 12–1 月是旺季，每 4 小时一次。 -- 同一个网站同时只进行一次普通请求、附件下载或完整的浏览器页面访问；一次完成后等 5 秒，再开始下一次。浏览器的一次访问包含同站的脚本挑战、重载和页面子资源，子资源按浏览器正常方式加载；跨站跳转的排队规则见[开发约定 §2](development.md#2-抓取和判别)。请求头写明是鼠钟，附上联系方式。 +- 同一个网站同一时间只发一个请求，两次请求之间隔 5 秒；请求头写明是鼠钟，附上联系方式。浏览器的一次页面访问算一个请求。 - 不读 robots.txt：抓的都是公开的通知列表，礼貌靠上面这几条保证；有的学院官网一请求 robots.txt 就断开连接（比如西电），照它的规矩反而一页都抓不了。 - 失败了等下一轮再抓，一轮里不反复重试。 @@ -169,7 +169,7 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 - **新通知**：列表里出现没见过的链接，直接比网址。 - **正文变化**：还没截止的项目，每天重抓一次它的通知，在它的来源那一轮里一起抓；正文的文字和上一份快照不同，就存一份新快照，再交给抽取。 - **删除**：重抓时通知打不开（404、410），标成「原文已删除」，快照照样能看。从列表里消失不算：列表会翻页，旧通知本来就会被挤下去。 -- **附件**：PDF、Word、Excel 和图片都下载下来，解析出文字，图片做文字识别；附件和正文一起存进快照。单个附件下载失败不影响通知：保留名称、原下载链接和失败原因；需要验证码的明确标注，不破解。接口返回的短错误提示作为原因保存，不作为附件正文或文件。最新快照仍有下载错误或缺文件的通知，即使抽取没有项目，也在来源后续轮次里按一天一次的间隔重抓。 +- **附件**：PDF、Word、Excel 和图片都下载下来，解析出文字，图片做文字识别；附件和正文一起存进快照。 ### 7.5 抓取看板 diff --git a/docs/development.md b/docs/development.md index e957a29..668d8c3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -59,16 +59,11 @@ crawler/ 抓取节点 - 接入来源时按通用抓取 → 配置 → 浏览器抓取 → 专用适配器的顺序试，前一种能抓就不用后一种（[总体设计 §7.2](design.md#72-怎么抓)）。 - 建站系统的模板放在代码里，和它的测试在一起；单个来源的配置存在网站的数据库里，在管理页面改，抓取节点领活时一起拿到。 -- 所有对来源网站的访问都经过 `crawl/fetch.py` 的 `Fetcher`，整个进程共用一个，[总体设计 §7.3](design.md#73-频率和礼貌)的规矩由它保证：UA 写明鼠钟；普通 HTTP 请求、完整的浏览器页面访问和程序下载的附件共用按网站排队的许可，多个任务同时抓也一样；上一次访问完成并释放许可后，再等 5 秒才开始下一次。 -- 浏览器的 `get(url)` 在顶层导航前取得网站许可，持有到页面关闭，覆盖同站的 HTTP 跳转、脚本挑战和重载；页面子资源按浏览器正常方式加载。顶层导航跨 host 时，先释放原站许可，再等待目标站的许可。普通 HTTP 请求和附件的跳转每一步单独取得许可，跳到别的网站按那个网站的间隔排队。 -- `crawl_source` 每轮通过 `fetcher.session(source.browser)` 创建一个 `PageSession`，交给 `Steps.list_items` 和 `Steps.read`：`get(url)` 取页面，`download(url, page_url=原文网址)` 下载附件，两者都返回 `Response`。普通来源直接请求；`browser` 为真的来源用浏览器取渲染后的 HTML，以 UTF-8 字节返回。列表、正文和附件的解析仍由对应步骤负责。 -- 读取步骤用 `crawl/attachments.py` 的 `download_attachment` 下载附件：识别验证码、HTTP 和传输错误、接口返回的 HTML 错误提示，失败时返回原链接和独立的 `error`，文件为空；成功时保留原始字节，文字交给附件解析。错误页不上传为文件，错误原因不进入抽取用的 `text`，也不把整条通知标成已删除。日志只记下载失败和链接，不写源站返回的文字。 +- 所有对来源网站的请求都经过 `crawl/fetch.py` 的 `Fetcher`，整个进程共用一个。UA 和请求间隔遵守[总体设计 §7.3](design.md#73-频率和礼貌)，多个任务也不能同时请求同一个网站。普通 HTTP 请求和附件下载逐次处理跳转，跳到别的网站就按那个网站的间隔排队。 +- 浏览器一次页面访问取一次许可，拿到页面关闭。页面子资源正常加载，跨网站跳转沿用起始网站的许可。 - 浏览器抓取启动时关掉 Chromium 的自动化标记（`--disable-blink-features=AutomationControlled`），UA 用浏览器自己的，把 `HeadlessChrome` 写成 `Chrome`，末尾加上鼠钟的标识；瑞数这类脚本挑战两处都检查（[总体设计 §5.2](design.md#52-实测)）。不冒充别的浏览器和系统。 -- 浏览器来源每一轮共用一个无痕会话，列表、通知、跨 host 跳转和附件沿用它，抓完就关，失败或取消也关闭；不同来源、不同轮次不共享会话。Cookie 由浏览器按 domain、path 和 secure 规则筛选；附件使用原文所在会话的请求客户端下载，共享浏览器管理的 Cookie。 -- 整个进程共用一个 Chromium 无头精简版，首次读取浏览器页面时才启动；同时最多开 2 个页面，多的排队。拒绝弹窗和浏览器下载；不加载图片、字体和音视频，省对方的流量，也省内存。 -- 浏览器页面等到最终文档已提交并触发 `DOMContentLoaded` 后读取 HTML，并确认读取期间未开始新的主文档导航。快照对应这一时刻的 DOM，列表和正文是否有效由解析步骤检查。 -- 普通 HTTP 请求（含普通来源的附件）使用 httpx 的网络操作超时：连接、连接池等待、每次读取和写入最多各等 30 秒，不是整次传输的总期限。浏览器来源的附件每个跳转请求总期限为 30 秒，从取得许可后开始，不含等待站点许可和间隔。 -- 浏览器页面总期限为 300 秒，从取得页面名额后开始，包含页面创建、站点许可排队、脚本挑战刷新和 DOM 就绪；等待页面名额不计入。期限内等不到最终页面就按超时记录，不把最初的 202、412 挑战页当成成功结果。成功、失败或取消后都先回收已发出的创建操作及页面，再释放网站许可和页面名额。 +- 每个浏览器来源每轮使用一个无痕会话，列表、正文和附件共用其中的 Cookie，抓完就关。脚本挑战设好 Cookie 后，后面的页面不用再过挑战；不同来源、不同轮次不共享会话。 +- 浏览器同时最多开 2 个页面，多的排队。拒绝弹窗和浏览器下载，附件由程序另外下载；不加载图片、字体和音视频，省对方的流量，也省内存。 - 解析和模型分工：`parse` 用程序取出能确定的东西，模型只做判断（[总体设计 §8.2](design.md#82-抽取)）。解析不猜：判断不了的交给模型或人，不硬配。 - 判别模型和 LLM 的提示词放在调用它的代码旁边。输出都按固定的结构解析，解析不了的当失败处理。 - 每一轮抓取都交一份抓取记录，失败也交，写明原因；不在一轮里反复重试。各步抓不成时抛 `FetchError`（`Fetcher` 抛的）或 `StepError`（写明失败类别，比如 `http_status` 和「HTTP 403」），一轮的流程把它记进抓取记录；别的异常是 bug，照常抛出。 @@ -80,7 +75,7 @@ uv sync uv run playwright install --with-deps --only-shell chromium ``` -第二条安装 Chromium 无头精简版及系统依赖。CI 不安装浏览器：自动测试使用假的浏览器事件、固定响应和假时钟，不连接真实网站；真实浏览器和来源网站的验证在本地进行，结果写进 PR。 +第二条安装 Chromium 无头精简版及系统依赖。CI 不装浏览器，自动测试用假的浏览器对象、固定响应和假时钟，不连真实网站。真实浏览器和来源网站在本地手动验证，结果写进 PR。 ### 2.1 列表配置 diff --git a/web/db/migrations/0001_init.sql b/web/db/migrations/0001_init.sql index 04f2329..257a762 100644 --- a/web/db/migrations/0001_init.sql +++ b/web/db/migrations/0001_init.sql @@ -141,7 +141,6 @@ CREATE TABLE attachments ( url text NOT NULL, content_type text NOT NULL, text text NOT NULL, -- extracted, or recognised in images - error text, -- why the attachment could not be downloaded data bytea, -- NULL until the crawler uploads the file PRIMARY KEY (snapshot_id, position) ); diff --git a/web/db/types.ts b/web/db/types.ts index 83832fe..b8a7cce 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -26,7 +26,6 @@ export type Timestamp = ColumnType; export interface Attachments { content_type: string; data: Buffer | null; - error: string | null; name: string; position: number; snapshot_id: number; diff --git a/web/openapi/crawler.json b/web/openapi/crawler.json index 83a94a7..0ed5c1c 100644 --- a/web/openapi/crawler.json +++ b/web/openapi/crawler.json @@ -191,7 +191,7 @@ "screen", "fetch" ], - "description": "screen：还没初筛；fetch：初筛通过但最新快照还没抽取的，或者项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的" + "description": "screen：还没初筛；fetch：要抓正文，初筛通过还没抽取的，或者项目还没截止、一天没抓过的" } }, "required": [ @@ -600,12 +600,6 @@ "text": { "type": "string", "maxLength": 2097152 - }, - "error": { - "type": "string", - "minLength": 1, - "maxLength": 2000, - "description": "附件下载失败原因；不写表示下载正常，不影响通知正文和其他附件" } }, "required": [ @@ -1558,16 +1552,6 @@ "required": false, "name": "position", "in": "path" - }, - { - "schema": { - "type": "string", - "maxLength": 2000, - "format": "uri" - }, - "required": true, - "name": "url", - "in": "query" } ], "requestBody": { diff --git a/web/server/modules/ingest/routes.test.ts b/web/server/modules/ingest/routes.test.ts index 8a683d8..e91771a 100644 --- a/web/server/modules/ingest/routes.test.ts +++ b/web/server/modules/ingest/routes.test.ts @@ -1,4 +1,3 @@ -import type { InferRequestType } from "hono/client"; import { testClient } from "hono/testing"; import { beforeAll, expect, test } from "vitest"; import { app } from "../../app.ts"; @@ -6,15 +5,6 @@ import { db } from "../../platform/db.ts"; import { createCrawlerToken } from "./service.ts"; const api = testClient(app).api.v1.crawler; -type AttachmentInfo = InferRequestType< - (typeof api.notices)[":id"]["snapshot"]["$post"] ->["json"]["attachments"][number]; -const file = { - name: "申请表.docx", - url: "https://x.edu.cn/f.docx", - content_type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - text: "表", -}; let crawler = { headers: {} }; @@ -55,15 +45,25 @@ async function list(sourceId: number, items: { url: string; title: string }[]) { return res.json(); } -async function snapshot(noticeId: number, attachments: AttachmentInfo[] = [], text = "正文") { +async function snapshot(noticeId: number, attachmentText?: string) { const res = await api.notices[":id"].snapshot.$post( { param: { id: String(noticeId) }, json: { fetched_at: 1780000000, html: "

正文

", - text, - attachments, + text: "正文", + attachments: + attachmentText === undefined + ? [] + : [ + { + name: "申请表.docx", + url: "https://x.edu.cn/f.docx", + content_type: "x", + text: attachmentText, + }, + ], }, }, crawler, @@ -222,289 +222,10 @@ test("the same words make no new snapshot, changed attachment text does", async const { pending } = await list(source.id, [{ url: "https://c.edu.cn/1.htm", title: "推免通知" }]); const noticeId = pending[0].id; - const first = await snapshot(noticeId, [file]); + const first = await snapshot(noticeId, "表"); expect(first).toMatchObject({ missing_files: [0], extraction: { colleges: [{}] } }); - expect(await snapshot(noticeId, [file])).toEqual(first); - expect((await snapshot(noticeId, [{ ...file, text: "新表" }])).snapshot_id).not.toBe( - first.snapshot_id, - ); -}); - -test("attachment download errors keep their links and do not require files", async () => { - const source = await newSource("attachments-error.edu.cn"); - const [notice] = ( - await list(source.id, [{ url: "https://attachments-error.edu.cn/1.htm", title: "通知" }]) - ).pending; - const attachments = [ - { ...file, text: "", error: "需要验证码" }, - { ...file, url: "https://x.edu.cn/deleted.docx", text: "", error: "获取文件错误" }, - ]; - const first = await snapshot(notice.id, attachments); - expect(first).toMatchObject({ missing_files: [], extraction: { colleges: [{}] } }); - expect(await snapshot(notice.id, attachments)).toEqual(first); - expect( - await db.selectFrom("snapshots").select("id").where("notice_id", "=", notice.id).execute(), - ).toHaveLength(1); - const stored = await db - .selectFrom("attachments") - .select(["position", "url", "text", "error", "data"]) - .where("snapshot_id", "=", first.snapshot_id) - .orderBy("position") - .execute(); - expect(stored).toEqual( - attachments.map((attachment, position) => ({ - position, - url: attachment.url, - text: "", - error: attachment.error, - data: null, - })), - ); -}); - -test("error updates and recovery reuse a text snapshot and request recovered files", async () => { - const source = await newSource("attachments-recovery.edu.cn"); - const [notice] = ( - await list(source.id, [{ url: "https://attachments-recovery.edu.cn/1.htm", title: "通知" }]) - ).pending; - const attachment = { ...file, text: "" }; - const first = await snapshot(notice.id, [{ ...attachment, error: "需要验证码" }]); - await db - .updateTable("snapshots") - .set({ extracted_at: new Date() }) - .where("id", "=", first.snapshot_id) - .execute(); - expect(await snapshot(notice.id, [{ ...attachment, error: "HTTP 503" }])).toEqual({ - snapshot_id: first.snapshot_id, - missing_files: [], - extraction: null, - }); - expect( - await db - .selectFrom("attachments") - .select("error") - .where("snapshot_id", "=", first.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ error: "HTTP 503" }); - expect(await snapshot(notice.id, [attachment])).toEqual({ - snapshot_id: first.snapshot_id, - missing_files: [0], - extraction: null, - }); - expect( - await db - .selectFrom("attachments") - .select("error") - .where("snapshot_id", "=", first.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ error: null }); - await db - .updateTable("attachments") - .set({ data: Buffer.from("original file") }) - .where("snapshot_id", "=", first.snapshot_id) - .execute(); - expect(await snapshot(notice.id, [attachment])).toEqual({ - snapshot_id: first.snapshot_id, - missing_files: [], - extraction: null, - }); -}); - -test("failed downloads preserve known attachment text and files without changing old snapshots", async () => { - const source = await newSource("attachments-known.edu.cn"); - const [notice] = ( - await list(source.id, [{ url: "https://attachments-known.edu.cn/1.htm", title: "通知" }]) - ).pending; - const first = await snapshot(notice.id, [file]); - const data = Buffer.from("original file"); - await db - .updateTable("attachments") - .set({ data }) - .where("snapshot_id", "=", first.snapshot_id) - .execute(); - await db - .updateTable("snapshots") - .set({ extracted_at: new Date() }) - .where("id", "=", first.snapshot_id) - .execute(); - const failed = { ...file, text: "", content_type: "text/html", error: "需要验证码" }; - expect(await snapshot(notice.id, [failed])).toEqual({ - snapshot_id: first.snapshot_id, - missing_files: [], - extraction: null, - }); - expect( - await db - .selectFrom("attachments") - .select(["text", "content_type", "error", "data"]) - .where("snapshot_id", "=", first.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ text: file.text, content_type: file.content_type, error: failed.error, data }); - const next = await snapshot(notice.id, [{ ...failed, error: "HTTP 503" }], "修改后的正文"); - expect(next.snapshot_id).not.toBe(first.snapshot_id); - expect(next.missing_files).toEqual([]); - expect( - await db - .selectFrom("attachments") - .select(["text", "error", "data"]) - .where("snapshot_id", "=", next.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ text: file.text, error: "HTTP 503", data }); - expect( - await db - .selectFrom("attachments") - .select("error") - .where("snapshot_id", "=", first.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ error: failed.error }); -}); - -test("attachment URL and order updates keep text versions and discard mismatched files", async () => { - const source = await newSource("attachments-order.edu.cn"); - const [notice] = ( - await list(source.id, [{ url: "https://attachments-order.edu.cn/1.htm", title: "通知" }]) - ).pending; - const attachments = [ - { ...file, text: "" }, - { ...file, url: "https://x.edu.cn/g.docx", text: "" }, - ]; - const first = await snapshot(notice.id, attachments); - await db - .updateTable("attachments") - .set({ data: Buffer.from("original file") }) - .where("snapshot_id", "=", first.snapshot_id) - .execute(); - const reordered = [{ ...attachments[1], error: "需要验证码" }, attachments[0]]; - expect(await snapshot(notice.id, reordered)).toMatchObject({ - snapshot_id: first.snapshot_id, - missing_files: [1], - }); - expect( - await db - .selectFrom("attachments") - .select(["position", "url", "error", "data"]) - .where("snapshot_id", "=", first.snapshot_id) - .orderBy("position") - .execute(), - ).toEqual( - reordered.map((attachment, position) => ({ - position, - url: attachment.url, - error: "error" in attachment ? attachment.error : null, - data: null, - })), - ); -}); - -test("a delayed attachment upload cannot overwrite a replacement URL", async () => { - const source = await newSource("attachments-stale.edu.cn"); - const [notice] = ( - await list(source.id, [{ url: "https://attachments-stale.edu.cn/1.htm", title: "通知" }]) - ).pending; - const original = { ...file, text: "" }; - const first = await snapshot(notice.id, [original]); - const replacement = { ...original, url: "https://x.edu.cn/replacement.docx" }; - expect(await snapshot(notice.id, [replacement])).toMatchObject({ - snapshot_id: first.snapshot_id, - missing_files: [0], - }); - const upload = (url: string, data: string) => - app.request( - `/api/v1/crawler/snapshots/${first.snapshot_id}/attachments/0?${new URLSearchParams({ url })}`, - { - method: "PUT", - headers: { ...crawler.headers, "content-type": "application/octet-stream" }, - body: data, - }, - ); - expect((await upload(original.url, "stale file")).status).toBe(404); - expect( - await db - .selectFrom("attachments") - .select("data") - .where("snapshot_id", "=", first.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ data: null }); - expect((await upload(replacement.url, "replacement file")).status).toBe(204); - expect( - await db - .selectFrom("attachments") - .select("data") - .where("snapshot_id", "=", first.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ data: Buffer.from("replacement file") }); -}); - -test("failed attachments are fetched once a day even when extraction found no programs", async () => { - const source = await newSource("attachments-retry.edu.cn"); - const items = [{ url: "https://attachments-retry.edu.cn/1.htm", title: "通知" }]; - const [notice] = (await list(source.id, items)).pending; - await api.sources[":id"].screening.$post( - { - param: { id: String(source.id) }, - json: { results: [{ notice_id: notice.id, score: 0.9, screened_in: true }], usage: [] }, - }, - crawler, - ); - const failed = { ...file, text: "", error: "HTTP 503" }; - const first = await snapshot(notice.id, [failed]); - const extracted = await api.snapshots[":id"].extraction.$post( - { param: { id: String(first.snapshot_id) }, json: { programs: [], usage: [] } }, - crawler, - ); - expect(extracted.status).toBe(204); - expect((await list(source.id, items)).pending).toEqual([]); - - const dayAgo = new Date(Date.now() - 25 * 3600_000); - await db.updateTable("notices").set({ checked_at: dayAgo }).where("id", "=", notice.id).execute(); - expect((await list(source.id, items)).pending).toMatchObject([{ id: notice.id, step: "fetch" }]); - - const recovered = await snapshot(notice.id, [{ ...file, text: "" }]); - expect(recovered).toEqual({ - snapshot_id: first.snapshot_id, - missing_files: [0], - extraction: null, - }); - await db.updateTable("notices").set({ checked_at: dayAgo }).where("id", "=", notice.id).execute(); - expect((await list(source.id, items)).pending).toMatchObject([{ id: notice.id, step: "fetch" }]); - const uploaded = await app.request( - `/api/v1/crawler/snapshots/${first.snapshot_id}/attachments/0?${new URLSearchParams({ url: file.url })}`, - { - method: "PUT", - headers: { ...crawler.headers, "content-type": "application/octet-stream" }, - body: "recovered file", - }, - ); - expect(uploaded.status).toBe(204); - expect((await list(source.id, items)).pending).toEqual([]); - - await snapshot(notice.id, [failed]); - const next = await snapshot(notice.id, [file]); - expect(next.snapshot_id).not.toBe(first.snapshot_id); - const nextUploaded = await app.request( - `/api/v1/crawler/snapshots/${next.snapshot_id}/attachments/0?${new URLSearchParams({ url: file.url })}`, - { - method: "PUT", - headers: { ...crawler.headers, "content-type": "application/octet-stream" }, - body: "new file", - }, - ); - expect(nextUploaded.status).toBe(204); - expect((await list(source.id, items)).pending).toMatchObject([{ id: notice.id, step: "fetch" }]); - const nextExtracted = await api.snapshots[":id"].extraction.$post( - { param: { id: String(next.snapshot_id) }, json: { programs: [], usage: [] } }, - crawler, - ); - expect(nextExtracted.status).toBe(204); - expect( - await db - .selectFrom("attachments") - .select("error") - .where("snapshot_id", "=", first.snapshot_id) - .executeTakeFirstOrThrow(), - ).toEqual({ error: failed.error }); - await db.updateTable("notices").set({ checked_at: dayAgo }).where("id", "=", notice.id).execute(); - expect((await list(source.id, items)).pending).toEqual([]); + expect(await snapshot(noticeId, "表")).toEqual(first); + expect((await snapshot(noticeId, "新表")).snapshot_id).not.toBe(first.snapshot_id); }); test("a notice of a program still open comes back to be fetched once a day", async () => { diff --git a/web/server/modules/ingest/routes.ts b/web/server/modules/ingest/routes.ts index bc03217..1dad207 100644 --- a/web/server/modules/ingest/routes.ts +++ b/web/server/modules/ingest/routes.ts @@ -130,7 +130,7 @@ const ListingResult = z published_on: IsoDate.nullable(), step: z.enum(["screen", "fetch"]).openapi({ description: - "screen：还没初筛；fetch：初筛通过但最新快照还没抽取的，或者项目还没截止、最新快照有附件下载失败或文件未补传，且一天没抓过的", + "screen：还没初筛；fetch：要抓正文，初筛通过还没抽取的，或者项目还没截止、一天没抓过的", }), }) .openapi("PendingNotice"), @@ -287,10 +287,6 @@ export const ingestRoutes = router url: Url, content_type: z.string().min(1).max(200), text: z.string().max(2 * MB), - error: z.string().min(1).max(2000).optional().openapi({ - description: - "附件下载失败原因；不写表示下载正常，不影响通知正文和其他附件", - }), }) .openapi("AttachmentInfo"), ) @@ -327,7 +323,6 @@ export const ingestRoutes = router security, request: { params: z.object({ id: IntParam, position: IntParam }), - query: z.object({ url: Url }), body: { content: { "application/octet-stream": { schema: z.string().openapi({ format: "binary" }) }, @@ -343,12 +338,7 @@ export const ingestRoutes = router }), async (c) => { const { id, position } = c.req.valid("param"); - await storeAttachmentFile( - id, - position, - c.req.valid("query").url, - Buffer.from(await c.req.arrayBuffer()), - ); + await storeAttachmentFile(id, position, Buffer.from(await c.req.arrayBuffer())); return c.body(null, 204); }, ) diff --git a/web/server/modules/ingest/service.ts b/web/server/modules/ingest/service.ts index c201494..3c9e39b 100644 --- a/web/server/modules/ingest/service.ts +++ b/web/server/modules/ingest/service.ts @@ -20,8 +20,8 @@ const LEASE_MS = 2 * 3600_000; const INTERVAL_MS = 12 * 3600_000; const BUSY_INTERVAL_MS = 4 * 3600_000; const BUSY_MONTHS = new Set([3, 4, 5, 6, 7, 8, 9, 12, 1]); -// Incomplete attachments and notices of programs still open are fetched daily (design §7.4), -// in their source's round; programs without a deadline are watched for this long after creation. +// Notices of programs still open are fetched again once a day to catch changes (design §7.4), +// in their source's round; those without a deadline, for this long after the program appeared. const RECHECK_MS = 24 * 3600_000; const UNDATED_RECHECK_MS = 90 * 24 * 3600_000; const MAX_SOURCES = 50; @@ -108,7 +108,7 @@ async function recordUsage( // Saves what the list showed and returns the source's notices that still need work: new ones // to screen; screened-in ones not yet extracted, which a round cut short leaves behind; and -// those with incomplete attachments or programs still open, not fetched for a day (design §7.4). +// those of programs still open, not fetched for a day (design §7.4). export async function recordListing(sourceId: number, items: ListedItem[]) { const now = new Date(); return db.transaction().execute(async (trx) => { @@ -131,53 +131,44 @@ export async function recordListing(sourceId: number, items: ListedItem[]) { .execute(); const pending = await trx .selectFrom("notices as n") - .leftJoinLateral( - (eb) => - eb - .selectFrom("snapshots") - .select(["id", "extracted_at"]) - .whereRef("notice_id", "=", "n.id") - .orderBy("id", "desc") - .limit(1) - .as("latest"), - (join) => join.onTrue(), - ) .select(["n.id", "n.url", "n.title", "n.published_on", "n.screening_score"]) .where("n.source_id", "=", sourceId) .where("n.removed_at", "is", null) .where((eb) => eb.or([ eb("n.screening_score", "is", null), - eb.and([eb("n.screened_in", "=", true), eb("latest.extracted_at", "is", null)]), eb.and([ - eb("n.checked_at", "<", new Date(now.getTime() - RECHECK_MS)), - eb.or([ + eb("n.screened_in", "=", true), + eb.not( eb.exists( eb .selectFrom("snapshots as s") - .innerJoin("program_versions as v", "v.snapshot_id", "s.id") - .innerJoin("programs as p", "p.id", "v.program_id") - .select("p.id") + .select("s.id") .whereRef("s.notice_id", "=", "n.id") - .where("p.deleted_at", "is", null) - .where((qb) => - qb.or([ - qb("p.apply_deadline", ">", now), - qb.and([ - qb("p.apply_deadline", "is", null), - qb("p.created_at", ">", new Date(now.getTime() - UNDATED_RECHECK_MS)), - ]), - ]), - ), - ), - eb.exists( - eb - .selectFrom("attachments as a") - .select("a.position") - .whereRef("a.snapshot_id", "=", "latest.id") - .where((qb) => qb.or([qb("a.error", "is not", null), qb("a.data", "is", null)])), + .where("s.extracted_at", "is not", null), ), - ]), + ), + ]), + eb.and([ + eb("n.checked_at", "<", new Date(now.getTime() - RECHECK_MS)), + eb.exists( + eb + .selectFrom("snapshots as s") + .innerJoin("program_versions as v", "v.snapshot_id", "s.id") + .innerJoin("programs as p", "p.id", "v.program_id") + .select("p.id") + .whereRef("s.notice_id", "=", "n.id") + .where("p.deleted_at", "is", null) + .where((qb) => + qb.or([ + qb("p.apply_deadline", ">", now), + qb.and([ + qb("p.apply_deadline", "is", null), + qb("p.created_at", ">", new Date(now.getTime() - UNDATED_RECHECK_MS)), + ]), + ]), + ), + ), ]), ]), ) @@ -212,13 +203,7 @@ export async function recordScreening( }); } -type AttachmentSubmission = { - name: string; - url: string; - content_type: string; - text: string; - error?: string; -}; +type AttachmentSubmission = { name: string; url: string; content_type: string; text: string }; // A new snapshot only when the words changed, the attachments' included (design §7.4). export async function recordSnapshot( @@ -241,28 +226,21 @@ export async function recordSnapshot( .orderBy("id", "desc") .limit(1) .executeTakeFirst(); - const latestAttachments = latest - ? await trx + const latestTexts = + latest && + ( + await trx .selectFrom("attachments") - .select(["url", "content_type", "text"]) + .select("text") .where("snapshot_id", "=", latest.id) .orderBy("position") .execute() - : []; - const attachments = submission.attachments.map((attachment, position) => { - const previous = latestAttachments.at(position); - // A failed download tells us nothing new about the file's existing words or type. - return attachment.error !== undefined && - attachment.text === "" && - previous?.url === attachment.url - ? { ...attachment, text: previous.text, content_type: previous.content_type } - : attachment; - }); + ).map((a) => a.text); const unchanged = latest !== undefined && latest.text === submission.text && - latestAttachments.length === attachments.length && - latestAttachments.every((attachment, i) => attachment.text === attachments[i].text); + latestTexts?.length === submission.attachments.length && + latestTexts.every((text, i) => text === submission.attachments[i].text); const snapshot = unchanged ? latest @@ -276,43 +254,17 @@ export async function recordSnapshot( }) .returning(["id", "extracted_at"]) .executeTakeFirstOrThrow(); - if (unchanged) { - for (const [position, attachment] of attachments.entries()) { - await trx - .updateTable("attachments") - .set({ - name: attachment.name, - url: attachment.url, - content_type: attachment.content_type, - error: attachment.error ?? null, - ...(latestAttachments[position].url !== attachment.url ? { data: null } : {}), - }) - .where("snapshot_id", "=", snapshot.id) - .where("position", "=", position) - .execute(); - } - } else if (attachments.length > 0) { + if (!unchanged && submission.attachments.length > 0) { await trx .insertInto("attachments") .values( - attachments.map((a, position) => ({ + submission.attachments.map((a, position) => ({ snapshot_id: snapshot.id, position, ...a, })), ) .execute(); - if (latest) { - await sql` - UPDATE attachments AS current SET data = previous.data - FROM attachments AS previous - WHERE current.snapshot_id = ${snapshot.id} - AND previous.snapshot_id = ${latest.id} - AND current.position = previous.position - AND current.url = previous.url AND current.text = previous.text - AND current.error IS NOT NULL - `.execute(trx); - } } await trx .updateTable("notices") @@ -325,7 +277,6 @@ export async function recordSnapshot( .select("position") .where("snapshot_id", "=", snapshot.id) .where("data", "is", null) - .where("error", "is", null) .orderBy("position") .execute(); return { @@ -385,18 +336,12 @@ async function extractionContext( return { published_on: publishedOn, colleges, programs }; } -export async function storeAttachmentFile( - snapshotId: number, - position: number, - url: string, - data: Buffer, -) { +export async function storeAttachmentFile(snapshotId: number, position: number, data: Buffer) { const result = await db .updateTable("attachments") .set({ data }) .where("snapshot_id", "=", snapshotId) .where("position", "=", position) - .where("url", "=", url) .executeTakeFirst(); if (result.numUpdatedRows === 0n) throw new AppError(404, "not_found", "附件不存在"); } From 338df6649b8f42c0cd64aa54786e9e8b48820dc2 Mon Sep 17 00:00:00 2001 From: lym12321 <41183806+lym12321@users.noreply.github.com> Date: Sat, 7 Nov 2026 14:01:11 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E7=AE=80=E5=8C=96=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E8=AE=B8=E5=8F=AF=E5=92=8C=E4=BC=9A=E8=AF=9D=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/src/crawler/crawl/browser.py | 73 +++++++++----------- crawler/src/crawler/crawl/browser_test.py | 28 ++++---- crawler/src/crawler/crawl/fetch.py | 81 +++++++++-------------- crawler/src/crawler/crawl/fetch_test.py | 13 ++-- crawler/src/crawler/crawl/rounds_test.py | 15 +++-- docs/development.md | 10 +-- 6 files changed, 99 insertions(+), 121 deletions(-) diff --git a/crawler/src/crawler/crawl/browser.py b/crawler/src/crawler/crawl/browser.py index 97603eb..970b542 100644 --- a/crawler/src/crawler/crawl/browser.py +++ b/crawler/src/crawler/crawl/browser.py @@ -1,7 +1,8 @@ import asyncio import logging import time -from collections.abc import Awaitable, Callable +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager from urllib.parse import urljoin from playwright.async_api import ( @@ -29,7 +30,6 @@ TIMEOUT, USER_AGENT, FetchError, - Permit, Response, ) @@ -42,7 +42,7 @@ class Browser: - def __init__(self, permit: Callable[[str], Awaitable[Permit]]) -> None: + def __init__(self, permit: Callable[[str], AbstractAsyncContextManager[None]]) -> None: self.permit = permit self.slots = asyncio.Semaphore(PAGE_LIMIT) self._lock = asyncio.Lock() @@ -59,11 +59,8 @@ async def _start(self) -> PlaywrightBrowser: browser = await playwright.chromium.launch( headless=True, args=["--disable-blink-features=AutomationControlled"] ) - page = await browser.new_page() - try: + async with await browser.new_page() as page: user_agent: str = await page.evaluate("navigator.userAgent") - finally: - await page.close() except BaseException: await playwright.stop() raise @@ -119,12 +116,10 @@ async def _route(self, route: Route) -> None: await route.continue_() async def get(self, url: str) -> Response: - async with self._owner.slots: - permit = await self._owner.permit(url) + async with self._owner.slots, self._owner.permit(url): started = time.monotonic() try: - page = await self._context.new_page() - try: + async with await self._context.new_page() as page: last_response: PlaywrightResponse | None = None def record_response(response: PlaywrightResponse) -> None: @@ -151,46 +146,40 @@ def record_response(response: PlaywrightResponse) -> None: time.monotonic() - started, ) return response - finally: - await page.close() except PlaywrightTimeoutError as error: raise FetchError("timeout", url) from error except PlaywrightError as error: raise FetchError("unreachable", url) from error - finally: - permit.release() async def download(self, url: str, *, page_url: str) -> Response: for _ in range(MAX_REDIRECTS + 1): - permit = await self._owner.permit(url) - try: - response = await self._context.request.get( - url, - headers={"Referer": page_url}, - timeout=TIMEOUT * 1000, - max_redirects=0, - max_retries=0, - fail_on_status_code=False, - ) + async with self._owner.permit(url): try: - location = response.headers.get("location") - if response.status in _REDIRECTS and location is not None: - url = urljoin(response.url, location) - continue - return Response( - response.url, - response.status, - response.headers.get("content-type"), - await response.body(), + response = await self._context.request.get( + url, + headers={"Referer": page_url}, + timeout=TIMEOUT * 1000, + max_redirects=0, + max_retries=0, + fail_on_status_code=False, ) - finally: - await response.dispose() - except PlaywrightTimeoutError as error: - raise FetchError("timeout", url) from error - except PlaywrightError as error: - raise FetchError("unreachable", url) from error - finally: - permit.release() + try: + location = response.headers.get("location") + if response.status in _REDIRECTS and location is not None: + url = urljoin(response.url, location) + continue + return Response( + response.url, + response.status, + response.headers.get("content-type"), + await response.body(), + ) + finally: + await response.dispose() + except PlaywrightTimeoutError as error: + raise FetchError("timeout", url) from error + except PlaywrightError as error: + raise FetchError("unreachable", url) from error raise FetchError("too_many_redirects", url) async def close(self) -> None: diff --git a/crawler/src/crawler/crawl/browser_test.py b/crawler/src/crawler/crawl/browser_test.py index 0574f87..1c55339 100644 --- a/crawler/src/crawler/crawl/browser_test.py +++ b/crawler/src/crawler/crawl/browser_test.py @@ -1,6 +1,7 @@ import asyncio from collections.abc import Awaitable, Callable from dataclasses import dataclass +from typing import Self import httpx import pytest @@ -74,7 +75,10 @@ async def goto(self, url: str, *, wait_until: str, **_options: object) -> None: async def content(self) -> str: return "保研通知" - async def close(self) -> None: + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *_: object) -> None: self.closed = True @@ -278,16 +282,15 @@ def test_last_main_navigation_response_is_used_without_a_cross_site_permit( ] async def main() -> None: - async with Fetcher() as fetcher: - destination_permit = await fetcher.permit("https://b/busy") - try: - async with fetcher.session(True, url="https://a/list") as session: - response = await session.get("https://a/notice") - assert response.status == 200 - assert response.url == "https://b/notice" - assert response.body.decode() == "保研通知" - finally: - destination_permit.release() + async with ( + Fetcher() as fetcher, + fetcher.permit("https://b/busy"), + fetcher.session(True, url="https://a/list") as session, + ): + response = await session.get("https://a/notice") + assert response.status == 200 + assert response.url == "https://b/notice" + assert response.body.decode() == "保研通知" run(main) @@ -368,7 +371,8 @@ async def fail(*_args: object, **_options: object) -> None: async def main() -> None: async with Fetcher() as fetcher: with pytest.raises(FetchError) as caught: - await fetcher.browser().session("https://a/list") + async with fetcher.session(True, url="https://a/list"): + pass assert caught.value.failure == failure assert caught.value.url == "https://a/list" assert all(context.closed for context in driver.contexts) diff --git a/crawler/src/crawler/crawl/fetch.py b/crawler/src/crawler/crawl/fetch.py index 540769a..dbbf1ad 100644 --- a/crawler/src/crawler/crawl/fetch.py +++ b/crawler/src/crawler/crawl/fetch.py @@ -3,7 +3,8 @@ import asyncio import time -from collections.abc import Awaitable, Callable +from collections.abc import AsyncGenerator, Awaitable, Callable +from contextlib import asynccontextmanager from dataclasses import dataclass, field from typing import TYPE_CHECKING, Literal, Self @@ -42,16 +43,6 @@ class _Site: finished_at: float = float("-inf") -@dataclass -class Permit: - _site: _Site - _clock: Callable[[], float] - - def release(self) -> None: - self._site.finished_at = self._clock() - self._site.lock.release() - - class Fetcher: """One per crawler process, shared by everything that crawls, so the rules hold across tasks.""" @@ -79,28 +70,33 @@ async def __aexit__(self, *_: object) -> None: finally: await self._client.aclose() - def session(self, browser: bool, *, url: str) -> PageSession: - return PageSession(self, browser, url) + @asynccontextmanager + async def session(self, browser: bool, *, url: str) -> AsyncGenerator[PageSession]: + browser_session = None + if browser: + if self._browser is None: + # Keep the browser library unloaded until a source actually needs it. + from crawler.crawl.browser import Browser - def browser(self) -> Browser: - if self._browser is None: - # Keep the browser library unloaded until a source actually needs it. - from crawler.crawl.browser import Browser - - self._browser = Browser(self.permit) - return self._browser + self._browser = Browser(self.permit) + browser_session = await self._browser.session(url) + try: + yield PageSession(self, browser_session) + finally: + if browser_session is not None: + await browser_session.close() - async def permit(self, url: str) -> Permit: + @asynccontextmanager + async def permit(self, url: str) -> AsyncGenerator[None]: site = self._sites.setdefault(httpx.URL(url).netloc.decode("ascii"), _Site()) - await site.lock.acquire() - try: + async with site.lock: wait = site.finished_at + INTERVAL - self._clock() if wait > 0: await self._sleep(wait) - return Permit(site, self._clock) - except BaseException: - site.lock.release() - raise + try: + yield + finally: + site.finished_at = self._clock() async def get(self, url: str) -> Response: # Each hop on its own, so a redirect to another site waits its turn there too. @@ -117,32 +113,19 @@ async def get(self, url: str) -> Response: raise FetchError("too_many_redirects", url) async def _send(self, url: str) -> httpx.Response: - permit = await self.permit(url) - try: - return await self._client.get(url) - except httpx.TimeoutException as error: - raise FetchError("timeout", url) from error - except httpx.TransportError as error: - raise FetchError("unreachable", url) from error - finally: - permit.release() + async with self.permit(url): + try: + return await self._client.get(url) + except httpx.TimeoutException as error: + raise FetchError("timeout", url) from error + except httpx.TransportError as error: + raise FetchError("unreachable", url) from error class PageSession: - def __init__(self, fetcher: Fetcher, browser: bool, url: str) -> None: + def __init__(self, fetcher: Fetcher, browser_session: BrowserSession | None) -> None: self._fetcher = fetcher - self._use_browser = browser - self._url = url - self._browser_session: BrowserSession | None = None - - async def __aenter__(self) -> Self: - if self._use_browser: - self._browser_session = await self._fetcher.browser().session(self._url) - return self - - async def __aexit__(self, *_: object) -> None: - if self._browser_session is not None: - await self._browser_session.close() + self._browser_session = browser_session async def get(self, url: str) -> Response: if self._browser_session is not None: diff --git a/crawler/src/crawler/crawl/fetch_test.py b/crawler/src/crawler/crawl/fetch_test.py index 2522dba..9171c5c 100644 --- a/crawler/src/crawler/crawl/fetch_test.py +++ b/crawler/src/crawler/crawl/fetch_test.py @@ -102,19 +102,18 @@ async def sleep(seconds: float) -> None: await asyncio.Future[None]() sites.now += seconds - async with Fetcher(clock=sites.clock, sleep=sleep) as fetcher: - first = await fetcher.permit("http://a/first") - first.release() - queued = asyncio.create_task(fetcher.permit("http://a/cancelled")) + async with Fetcher(httpx.MockTransport(sites.handle), sites.clock, sleep) as fetcher: + await fetcher.get("http://a/first") + queued = asyncio.create_task(fetcher.get("http://a/cancelled")) await waiting.wait() + sites.now = 2 queued.cancel() with pytest.raises(asyncio.CancelledError): await queued block = False async with asyncio.timeout(1): - next_request = await fetcher.permit("http://a/next") - next_request.release() - assert waits == [INTERVAL, INTERVAL] + await fetcher.get("http://a/next") + assert waits == [INTERVAL, INTERVAL - 2] asyncio.run(run()) diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index a54b3fe..0a95aff 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -64,14 +64,14 @@ def handle(self, request: httpx.Request) -> httpx.Response: ) return httpx.Response(204) - def run(self, steps: Steps) -> None: + def run(self, steps: Steps, source: WorkSource = SOURCE) -> None: async def main() -> None: nowhere = httpx.MockTransport(lambda _: httpx.Response(500)) async with ( Fetcher(nowhere) as fetcher, Website("http://site", "token", httpx.MockTransport(self.handle)) as website, ): - await crawl_source(SOURCE, fetcher, website, steps, clock=lambda: 1_800_000_000) + await crawl_source(source, fetcher, website, steps, clock=lambda: 1_800_000_000) asyncio.run(main()) @@ -215,14 +215,17 @@ async def unused(*_: object) -> Any: def test_a_session_that_cant_start_is_reported(monkeypatch: pytest.MonkeyPatch) -> None: site = FakeWebsite([], extracted=frozenset()) - async def unavailable(_pages: PageSession) -> PageSession: - raise FetchError("unreachable", SOURCE.url) + async def unavailable(_browser: object, url: str) -> None: + raise FetchError("unreachable", url) async def unused_read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: raise AssertionError("Listing failed before notices could be read") - monkeypatch.setattr(PageSession, "__aenter__", unavailable) - site.run(Steps(list_items, screen, unused_read, extract)) + monkeypatch.setattr("crawler.crawl.browser.Browser.session", unavailable) + site.run( + Steps(list_items, screen, unused_read, extract), + SOURCE.model_copy(update={"browser": True}), + ) assert [call for call, _ in site.calls] == ["POST /runs"] [run] = site.called("POST /runs") diff --git a/docs/development.md b/docs/development.md index 668d8c3..794674c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -59,23 +59,23 @@ crawler/ 抓取节点 - 接入来源时按通用抓取 → 配置 → 浏览器抓取 → 专用适配器的顺序试，前一种能抓就不用后一种（[总体设计 §7.2](design.md#72-怎么抓)）。 - 建站系统的模板放在代码里，和它的测试在一起；单个来源的配置存在网站的数据库里，在管理页面改，抓取节点领活时一起拿到。 -- 所有对来源网站的请求都经过 `crawl/fetch.py` 的 `Fetcher`，整个进程共用一个。UA 和请求间隔遵守[总体设计 §7.3](design.md#73-频率和礼貌)，多个任务也不能同时请求同一个网站。普通 HTTP 请求和附件下载逐次处理跳转，跳到别的网站就按那个网站的间隔排队。 -- 浏览器一次页面访问取一次许可，拿到页面关闭。页面子资源正常加载，跨网站跳转沿用起始网站的许可。 +- 整个进程共用一个 `Fetcher`（`crawl/fetch.py`），所有来源请求都经过它，遵守[总体设计 §7.3](design.md#73-频率和礼貌)的 UA 和间隔规则。普通 HTTP 请求和附件下载的每次跳转，都按目标网站的间隔排队。 +- 浏览器一次页面访问取一次许可，持有到页面关闭。页面子资源正常加载，跨网站跳转沿用起始网站的许可。 - 浏览器抓取启动时关掉 Chromium 的自动化标记（`--disable-blink-features=AutomationControlled`），UA 用浏览器自己的，把 `HeadlessChrome` 写成 `Chrome`，末尾加上鼠钟的标识；瑞数这类脚本挑战两处都检查（[总体设计 §5.2](design.md#52-实测)）。不冒充别的浏览器和系统。 -- 每个浏览器来源每轮使用一个无痕会话，列表、正文和附件共用其中的 Cookie，抓完就关。脚本挑战设好 Cookie 后，后面的页面不用再过挑战；不同来源、不同轮次不共享会话。 +- 每个浏览器来源每轮新建一个无痕会话，列表、正文和附件共用其中的 Cookie，抓完就关。 - 浏览器同时最多开 2 个页面，多的排队。拒绝弹窗和浏览器下载，附件由程序另外下载；不加载图片、字体和音视频，省对方的流量，也省内存。 - 解析和模型分工：`parse` 用程序取出能确定的东西，模型只做判断（[总体设计 §8.2](design.md#82-抽取)）。解析不猜：判断不了的交给模型或人，不硬配。 - 判别模型和 LLM 的提示词放在调用它的代码旁边。输出都按固定的结构解析，解析不了的当失败处理。 - 每一轮抓取都交一份抓取记录，失败也交，写明原因；不在一轮里反复重试。各步抓不成时抛 `FetchError`（`Fetcher` 抛的）或 `StepError`（写明失败类别，比如 `http_status` 和「HTTP 403」），一轮的流程把它记进抓取记录；别的异常是 bug，照常抛出。 -本地使用浏览器抓取时，在 `crawler/` 运行： +在 `crawler/` 安装项目依赖、Chromium 无头精简版及系统依赖： ```sh uv sync uv run playwright install --with-deps --only-shell chromium ``` -第二条安装 Chromium 无头精简版及系统依赖。CI 不装浏览器，自动测试用假的浏览器对象、固定响应和假时钟，不连真实网站。真实浏览器和来源网站在本地手动验证，结果写进 PR。 +CI 不装浏览器。自动测试用假的浏览器对象、固定响应和假时钟；真实浏览器和来源网站在本地手动验证，结果写进 PR。 ### 2.1 列表配置 From 8858d1a43c24a0ae62d8b58ebcffafc0efda9fdd Mon Sep 17 00:00:00 2001 From: lym12321 <41183806+lym12321@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:39:34 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E7=90=86=E9=A1=BA=E6=B5=8F=E8=A7=88?= =?UTF-8?q?=E5=99=A8=E4=B8=8E=E8=AF=B7=E6=B1=82=E5=B1=82=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/src/crawler/crawl/browser.py | 88 +++++++++------ crawler/src/crawler/crawl/browser_test.py | 125 ++++++---------------- crawler/src/crawler/crawl/fetch.py | 44 +------- crawler/src/crawler/crawl/fetch_test.py | 71 +----------- crawler/src/crawler/crawl/listing.py | 6 +- crawler/src/crawler/crawl/rounds.py | 7 +- crawler/src/crawler/crawl/rounds_test.py | 12 ++- crawler/src/crawler/parse/listing_test.py | 30 +++--- crawler/uv.lock | 80 ++++++++++++++ docs/development.md | 6 +- 10 files changed, 203 insertions(+), 266 deletions(-) diff --git a/crawler/src/crawler/crawl/browser.py b/crawler/src/crawler/crawl/browser.py index 970b542..03aad38 100644 --- a/crawler/src/crawler/crawl/browser.py +++ b/crawler/src/crawler/crawl/browser.py @@ -1,8 +1,9 @@ import asyncio import logging import time -from collections.abc import Callable -from contextlib import AbstractAsyncContextManager +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from typing import Self from urllib.parse import urljoin from playwright.async_api import ( @@ -25,10 +26,12 @@ TimeoutError as PlaywrightTimeoutError, ) +from crawler.crawl.contract import WorkSource from crawler.crawl.fetch import ( MAX_REDIRECTS, TIMEOUT, USER_AGENT, + Fetcher, FetchError, Response, ) @@ -41,15 +44,33 @@ _logger = logging.getLogger(__name__) +async def _close_popup(page: Page) -> None: + if await page.opener() is not None: + await page.close() + + +async def _route(route: Route) -> None: + if route.request.resource_type in _BLOCKED: + await route.abort() + else: + await route.continue_() + + class Browser: - def __init__(self, permit: Callable[[str], AbstractAsyncContextManager[None]]) -> None: - self.permit = permit + def __init__(self, fetcher: Fetcher) -> None: + self.fetcher = fetcher self.slots = asyncio.Semaphore(PAGE_LIMIT) self._lock = asyncio.Lock() self._playwright: Playwright | None = None self._browser: PlaywrightBrowser | None = None self._user_agent = "" + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *_: object) -> None: + await self.close() + async def _start(self) -> PlaywrightBrowser: async with self._lock: if self._browser is not None: @@ -69,23 +90,23 @@ async def _start(self) -> PlaywrightBrowser: self._browser = browser return browser - async def session(self, url: str) -> BrowserSession: + @asynccontextmanager + async def session(self, source: WorkSource) -> AsyncGenerator[PageSession]: + if not source.browser: + yield PageSession(self.fetcher, None) + return try: browser = await self._start() - context = await browser.new_context( + async with await browser.new_context( user_agent=self._user_agent, service_workers="block", accept_downloads=False - ) - session = BrowserSession(self, context) - try: - await session.configure() - except BaseException: - await context.close() - raise - return session + ) as context: + await context.route("**/*", _route) + context.on("page", _close_popup) + yield PageSession(self.fetcher, BrowserSession(self, context)) except PlaywrightTimeoutError as error: - raise FetchError("timeout", url) from error + raise FetchError("timeout", source.url) from error except PlaywrightError as error: - raise FetchError("unreachable", url) from error + raise FetchError("unreachable", source.url) from error async def close(self) -> None: try: @@ -101,22 +122,8 @@ def __init__(self, owner: Browser, context: BrowserContext) -> None: self._owner = owner self._context = context - async def configure(self) -> None: - await self._context.route("**/*", self._route) - self._context.on("page", self._new_page) - - async def _new_page(self, page: Page) -> None: - if await page.opener() is not None: - await page.close() - - async def _route(self, route: Route) -> None: - if route.request.resource_type in _BLOCKED: - await route.abort() - else: - await route.continue_() - async def get(self, url: str) -> Response: - async with self._owner.slots, self._owner.permit(url): + async with self._owner.slots, self._owner.fetcher.permit(url): started = time.monotonic() try: async with await self._context.new_page() as page: @@ -153,7 +160,7 @@ def record_response(response: PlaywrightResponse) -> None: async def download(self, url: str, *, page_url: str) -> Response: for _ in range(MAX_REDIRECTS + 1): - async with self._owner.permit(url): + async with self._owner.fetcher.permit(url): try: response = await self._context.request.get( url, @@ -182,5 +189,18 @@ async def download(self, url: str, *, page_url: str) -> Response: raise FetchError("unreachable", url) from error raise FetchError("too_many_redirects", url) - async def close(self) -> None: - await self._context.close() + +class PageSession: + def __init__(self, fetcher: Fetcher, browser_session: BrowserSession | None) -> None: + self._fetcher = fetcher + self._browser_session = browser_session + + async def get(self, url: str) -> Response: + if self._browser_session is not None: + return await self._browser_session.get(url) + return await self._fetcher.get(url) + + async def download(self, url: str, *, page_url: str) -> Response: + if self._browser_session is not None: + return await self._browser_session.download(url, page_url=page_url) + return await self._fetcher.get(url) diff --git a/crawler/src/crawler/crawl/browser_test.py b/crawler/src/crawler/crawl/browser_test.py index 1c55339..682c3c8 100644 --- a/crawler/src/crawler/crawl/browser_test.py +++ b/crawler/src/crawler/crawl/browser_test.py @@ -5,11 +5,14 @@ import httpx import pytest -from playwright.async_api import Error as PlaywrightError -from playwright.async_api import TimeoutError as PlaywrightTimeoutError from crawler.crawl.browser import Browser -from crawler.crawl.fetch import INTERVAL, Failure, Fetcher, FetchError +from crawler.crawl.contract import WorkSource +from crawler.crawl.fetch import INTERVAL, Fetcher, FetchError + +SOURCE = WorkSource( + id=7, url="https://a/list", section="通知公告", browser=True, config=None, backfill=False +) def run(main: Callable[[], Awaitable[None]]) -> None: @@ -66,8 +69,6 @@ async def goto(self, url: str, *, wait_until: str, **_options: object) -> None: driver.started.put_nowait(self) if driver.block_pages: await self.ready.wait() - if driver.error is not None: - raise driver.error for status, navigation, main_frame in driver.responses: frame = self.main_frame if main_frame else object() self.handlers["response"](FakeResponse(status, FakeRequest(frame, navigation))) @@ -103,8 +104,6 @@ def __init__(self, context: FakeContext) -> None: self.responses: list[FakeDownload] = [] async def get(self, url: str, *, headers: dict[str, str], **_options: object) -> FakeDownload: - if self.context.driver.error is not None: - raise self.context.driver.error response = FakeDownload( url, f"%PDF context {self.context.number}: {headers['Referer']}".encode() ) @@ -131,7 +130,10 @@ async def new_page(self) -> FakePage: self.pages.append(page) return page - async def close(self) -> None: + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *_: object) -> None: self.closed = True @@ -141,7 +143,6 @@ def __init__(self) -> None: self.started: asyncio.Queue[FakePage] = asyncio.Queue() self.block_pages = False self.destination: str | None = None - self.error: PlaywrightError | None = None self.responses = [(200, True, True)] async def new_context(self, **_options: object) -> FakeContext: @@ -177,7 +178,8 @@ async def handle(_request: httpx.Request) -> httpx.Response: async def main() -> None: async with ( Fetcher(httpx.MockTransport(handle), clock.clock, clock.sleep) as fetcher, - fetcher.session(True, url="https://a/list") as session, + Browser(fetcher) as browser, + browser.session(SOURCE) as session, ): first_http = asyncio.create_task(fetcher.get("https://a/first")) await http_started.wait() @@ -208,8 +210,9 @@ def test_page_limit_is_shared_by_rounds(driver: FakeBrowser) -> None: async def main() -> None: async with ( Fetcher() as fetcher, - fetcher.session(True, url="https://a/list") as first, - fetcher.session(True, url="https://b/list") as second, + Browser(fetcher) as browser, + browser.session(SOURCE) as first, + browser.session(SOURCE.model_copy(update={"id": 8, "url": "https://b/list"})) as second, ): tasks = [ asyncio.create_task(first.get("https://a/list")), @@ -235,17 +238,20 @@ def test_each_round_opens_reuses_and_closes_its_context(driver: FakeBrowser) -> clock = FakeClock() async def main() -> None: - async with Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher: - async with fetcher.session(False, url="https://a/list"): + async with ( + Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher, + Browser(fetcher) as browser, + ): + async with browser.session(SOURCE.model_copy(update={"browser": False})): assert driver.contexts == [] - async with fetcher.session(True, url="https://a/list") as session: + async with browser.session(SOURCE) as session: assert len(driver.contexts) == 1 await session.get("https://a/list") await session.get("https://a/notice") assert len(driver.contexts[0].pages) == 2 assert not driver.contexts[0].closed assert driver.contexts[0].closed - async with fetcher.session(True, url="https://a/list"): + async with browser.session(SOURCE): assert len(driver.contexts) == 2 assert driver.contexts[1].closed @@ -256,9 +262,12 @@ def test_downloads_use_the_rounds_context_request_client(driver: FakeBrowser) -> clock = FakeClock() async def main() -> None: - async with Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher: + async with ( + Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher, + Browser(fetcher) as browser, + ): for number in (1, 2): - async with fetcher.session(True, url="https://a/list") as session: + async with browser.session(SOURCE) as session: await session.get("https://a/notice") attachment = await session.download( "https://a/file.pdf", page_url="https://a/notice" @@ -284,8 +293,9 @@ def test_last_main_navigation_response_is_used_without_a_cross_site_permit( async def main() -> None: async with ( Fetcher() as fetcher, + Browser(fetcher) as browser, fetcher.permit("https://b/busy"), - fetcher.session(True, url="https://a/list") as session, + browser.session(SOURCE) as session, ): response = await session.get("https://a/notice") assert response.status == 200 @@ -300,81 +310,14 @@ def test_unresolved_challenges_fail(driver: FakeBrowser, status: int) -> None: driver.responses = [(status, True, True)] async def main() -> None: - async with Fetcher() as fetcher, fetcher.session(True, url="https://a/list") as session: + async with ( + Fetcher() as fetcher, + Browser(fetcher) as browser, + browser.session(SOURCE) as session, + ): with pytest.raises(FetchError) as caught: await session.get("https://a/list") assert caught.value.failure == "unreachable" assert driver.contexts[0].pages[0].closed run(main) - - -@pytest.mark.parametrize( - ("error", "failure"), - [ - (PlaywrightTimeoutError("Timed out"), "timeout"), - (PlaywrightError("Browser request failed"), "unreachable"), - ], -) -@pytest.mark.parametrize("download", [False, True]) -def test_playwright_errors_become_fetch_errors( - driver: FakeBrowser, error: PlaywrightError, failure: Failure, *, download: bool -) -> None: - driver.error = error - clock = FakeClock() - - async def main() -> None: - async with ( - Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher, - fetcher.session(True, url="https://a/list") as session, - ): - operation = ( - session.download("https://a/file.pdf", page_url="https://a/notice") - if download - else session.get("https://a/notice") - ) - with pytest.raises(FetchError) as caught: - await operation - assert caught.value.failure == failure - driver.error = None - assert (await session.get("https://a/retry")).status == 200 - assert clock.now == INTERVAL - - run(main) - - -@pytest.mark.parametrize("stage", ["start", "context", "configure"]) -@pytest.mark.parametrize( - ("error", "failure"), - [ - (PlaywrightTimeoutError("Timed out"), "timeout"), - (PlaywrightError("Browser initialization failed"), "unreachable"), - ], -) -def test_initialization_errors_include_the_source_url( - monkeypatch: pytest.MonkeyPatch, - driver: FakeBrowser, - stage: str, - error: PlaywrightError, - failure: Failure, -) -> None: - async def fail(*_args: object, **_options: object) -> None: - raise error - - if stage == "start": - monkeypatch.setattr(Browser, "_start", fail) - elif stage == "context": - monkeypatch.setattr(FakeBrowser, "new_context", fail) - else: - monkeypatch.setattr(FakeContext, "route", fail) - - async def main() -> None: - async with Fetcher() as fetcher: - with pytest.raises(FetchError) as caught: - async with fetcher.session(True, url="https://a/list"): - pass - assert caught.value.failure == failure - assert caught.value.url == "https://a/list" - assert all(context.closed for context in driver.contexts) - - run(main) diff --git a/crawler/src/crawler/crawl/fetch.py b/crawler/src/crawler/crawl/fetch.py index dbbf1ad..742c017 100644 --- a/crawler/src/crawler/crawl/fetch.py +++ b/crawler/src/crawler/crawl/fetch.py @@ -6,13 +6,10 @@ from collections.abc import AsyncGenerator, Awaitable, Callable from contextlib import asynccontextmanager from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Literal, Self +from typing import Literal, Self import httpx -if TYPE_CHECKING: - from crawler.crawl.browser import Browser, BrowserSession - USER_AGENT = "mouseclock/0.1 (+https://github.com/mousebaoyan/mouseclock)" # Seconds from the end of one page visit or HTTP request to the start of the next. INTERVAL = 5.0 @@ -58,33 +55,12 @@ def __init__( self._clock = clock self._sleep = sleep self._sites: dict[str, _Site] = {} - self._browser: Browser | None = None async def __aenter__(self) -> Self: return self async def __aexit__(self, *_: object) -> None: - try: - if self._browser is not None: - await self._browser.close() - finally: - await self._client.aclose() - - @asynccontextmanager - async def session(self, browser: bool, *, url: str) -> AsyncGenerator[PageSession]: - browser_session = None - if browser: - if self._browser is None: - # Keep the browser library unloaded until a source actually needs it. - from crawler.crawl.browser import Browser - - self._browser = Browser(self.permit) - browser_session = await self._browser.session(url) - try: - yield PageSession(self, browser_session) - finally: - if browser_session is not None: - await browser_session.close() + await self._client.aclose() @asynccontextmanager async def permit(self, url: str) -> AsyncGenerator[None]: @@ -120,19 +96,3 @@ async def _send(self, url: str) -> httpx.Response: raise FetchError("timeout", url) from error except httpx.TransportError as error: raise FetchError("unreachable", url) from error - - -class PageSession: - def __init__(self, fetcher: Fetcher, browser_session: BrowserSession | None) -> None: - self._fetcher = fetcher - self._browser_session = browser_session - - async def get(self, url: str) -> Response: - if self._browser_session is not None: - return await self._browser_session.get(url) - return await self._fetcher.get(url) - - async def download(self, url: str, *, page_url: str) -> Response: - if self._browser_session is not None: - return await self._browser_session.download(url, page_url=page_url) - return await self._fetcher.get(url) diff --git a/crawler/src/crawler/crawl/fetch_test.py b/crawler/src/crawler/crawl/fetch_test.py index 9171c5c..01de10d 100644 --- a/crawler/src/crawler/crawl/fetch_test.py +++ b/crawler/src/crawler/crawl/fetch_test.py @@ -5,9 +5,8 @@ from urllib.parse import urlsplit import httpx -import pytest -from crawler.crawl.fetch import INTERVAL, USER_AGENT, Fetcher, FetchError +from crawler.crawl.fetch import INTERVAL, USER_AGENT, Fetcher class FakeSites: @@ -86,71 +85,3 @@ def test_a_redirect_to_another_site_waits_its_turn_there() -> None: assert moved.url == "http://b/new" assert len(sites.times("b")) == 2 assert all(gap >= INTERVAL for gap in gaps(sites.times("b"))) - - -def test_cancelling_the_interval_wait_does_not_mark_a_request_finished() -> None: - async def run() -> None: - sites = FakeSites({}) - waiting = asyncio.Event() - waits: list[float] = [] - block = True - - async def sleep(seconds: float) -> None: - waits.append(seconds) - if block: - waiting.set() - await asyncio.Future[None]() - sites.now += seconds - - async with Fetcher(httpx.MockTransport(sites.handle), sites.clock, sleep) as fetcher: - await fetcher.get("http://a/first") - queued = asyncio.create_task(fetcher.get("http://a/cancelled")) - await waiting.wait() - sites.now = 2 - queued.cancel() - with pytest.raises(asyncio.CancelledError): - await queued - block = False - async with asyncio.timeout(1): - await fetcher.get("http://a/next") - assert waits == [INTERVAL, INTERVAL - 2] - - asyncio.run(run()) - - -@pytest.mark.parametrize("failure", ["timeout", "unreachable", "cancelled"]) -def test_failed_or_cancelled_http_requests_release_the_shared_permission(failure: str) -> None: - async def run() -> None: - sites = FakeSites({}) - started = asyncio.Event() - requests = 0 - - async def handle(request: httpx.Request) -> httpx.Response: - nonlocal requests - requests += 1 - if requests == 1: - started.set() - if failure == "cancelled": - await asyncio.Future[None]() - if failure == "timeout": - raise httpx.ReadTimeout("Timed out", request=request) - raise httpx.ConnectError("Connection failed", request=request) - return httpx.Response(200) - - async with Fetcher(httpx.MockTransport(handle), sites.clock, sites.sleep) as fetcher: - task = asyncio.create_task(fetcher.get("http://a/first")) - await started.wait() - if failure == "cancelled": - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - else: - with pytest.raises(FetchError) as caught: - await task - assert caught.value.failure == failure - async with asyncio.timeout(1): - response = await fetcher.get("http://a/next") - assert response.status == 200 - assert sites.now == INTERVAL - - asyncio.run(run()) diff --git a/crawler/src/crawler/crawl/listing.py b/crawler/src/crawler/crawl/listing.py index 4eb1bdb..0c51853 100644 --- a/crawler/src/crawler/crawl/listing.py +++ b/crawler/src/crawler/crawl/listing.py @@ -6,14 +6,14 @@ from pydantic import ValidationError +from crawler.crawl.browser import PageSession from crawler.crawl.contract import ListedItem, WorkSource -from crawler.crawl.fetch import Fetcher from crawler.crawl.rounds import StepError from crawler.parse.listing import ListConfig, parse_listing async def list_items( - fetcher: Fetcher, source: WorkSource, clock: Callable[[], float] = time.time + pages: PageSession, source: WorkSource, clock: Callable[[], float] = time.time ) -> list[ListedItem]: try: config = ListConfig.model_validate(source.config or {}) @@ -37,7 +37,7 @@ def request_url(url: str) -> str: if current in visited: break visited.add(current) - response = await fetcher.get(current) + response = await pages.get(current) visited.add(response.url) if not 200 <= response.status < 300: raise StepError("http_status", f"HTTP {response.status}") diff --git a/crawler/src/crawler/crawl/rounds.py b/crawler/src/crawler/crawl/rounds.py index 334bf21..8ea00e5 100644 --- a/crawler/src/crawler/crawl/rounds.py +++ b/crawler/src/crawler/crawl/rounds.py @@ -8,6 +8,7 @@ from contextlib import AsyncExitStack from dataclasses import dataclass +from crawler.crawl.browser import Browser, PageSession from crawler.crawl.contract import ( AttachmentInfo, ExtractedProgram, @@ -25,7 +26,7 @@ Usage, WorkSource, ) -from crawler.crawl.fetch import Fetcher, FetchError, PageSession +from crawler.crawl.fetch import FetchError from crawler.crawl.website import Website logger = logging.getLogger(__name__) @@ -80,7 +81,7 @@ def _failure(error: FetchError | StepError) -> tuple[Failure, str | None]: async def crawl_source( source: WorkSource, - fetcher: Fetcher, + browser: Browser, website: Website, steps: Steps, clock: Callable[[], float] = time.time, @@ -88,7 +89,7 @@ async def crawl_source( started_at = int(clock()) async with AsyncExitStack() as stack: try: - pages = await stack.enter_async_context(fetcher.session(source.browser, url=source.url)) + pages = await stack.enter_async_context(browser.session(source)) items = await steps.list_items(pages, source) if not items: raise StepError("no_items") diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index 0a95aff..4f54249 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -6,6 +6,7 @@ import httpx import pytest +from crawler.crawl.browser import Browser, PageSession from crawler.crawl.contract import ( AttachmentInfo, ExtractedFields, @@ -17,7 +18,7 @@ Usage, WorkSource, ) -from crawler.crawl.fetch import Fetcher, FetchError, PageSession +from crawler.crawl.fetch import Fetcher, FetchError from crawler.crawl.rounds import Attachment, NoticePage, StepError, Steps, crawl_source from crawler.crawl.website import Website @@ -69,9 +70,10 @@ async def main() -> None: nowhere = httpx.MockTransport(lambda _: httpx.Response(500)) async with ( Fetcher(nowhere) as fetcher, + Browser(fetcher) as browser, Website("http://site", "token", httpx.MockTransport(self.handle)) as website, ): - await crawl_source(source, fetcher, website, steps, clock=lambda: 1_800_000_000) + await crawl_source(source, browser, website, steps, clock=lambda: 1_800_000_000) asyncio.run(main()) @@ -215,13 +217,13 @@ async def unused(*_: object) -> Any: def test_a_session_that_cant_start_is_reported(monkeypatch: pytest.MonkeyPatch) -> None: site = FakeWebsite([], extracted=frozenset()) - async def unavailable(_browser: object, url: str) -> None: - raise FetchError("unreachable", url) + async def unavailable(_browser: Browser) -> None: + raise FetchError("unreachable", SOURCE.url) async def unused_read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: raise AssertionError("Listing failed before notices could be read") - monkeypatch.setattr("crawler.crawl.browser.Browser.session", unavailable) + monkeypatch.setattr(Browser, "_start", unavailable) site.run( Steps(list_items, screen, unused_read, extract), SOURCE.model_copy(update={"browser": True}), diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py index ba5da48..a37ce50 100644 --- a/crawler/src/crawler/parse/listing_test.py +++ b/crawler/src/crawler/parse/listing_test.py @@ -5,6 +5,7 @@ import pytest from pydantic import JsonValue +from crawler.crawl.browser import Browser, PageSession from crawler.crawl.contract import WorkSource from crawler.crawl.fetch import Fetcher from crawler.crawl.fetch_test import FakeSites @@ -107,7 +108,7 @@ def test_documented_http_configuration_reads_the_real_page_structure() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(config=config), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -144,7 +145,7 @@ def test_backfill_keeps_boundary_and_unknown_dates_and_ignores_repeated_pinned_r ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=backfill), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -168,7 +169,7 @@ def test_leap_day_backfill_uses_calendar_years() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2024, 2, 29, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -199,7 +200,7 @@ def test_numbered_pager_advances_one_page_and_stops_at_last() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -240,7 +241,7 @@ def test_explicit_http_config_and_redirect_base_are_used_for_pagination_and_link config["selectors"] = selectors result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True, config=config), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -274,15 +275,16 @@ def test_bad_configuration_is_recorded_by_the_round( sites = FakeSites({BASE: httpx.Response(200, content=page([("1", "2026-09-10")]))}) website = FakeWebsite([], frozenset()) - async def read(_fetcher: Fetcher, _source: WorkSource, _url: str) -> NoticePage: + async def read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: raise AssertionError("a failed list must not read notices") async def run(fetcher: Fetcher) -> None: - async with Website( - "http://site", "local-test", httpx.MockTransport(website.handle) - ) as client: + async with ( + Browser(fetcher) as browser, + Website("http://site", "local-test", httpx.MockTransport(website.handle)) as client, + ): await crawl_source( - source(config=config), fetcher, client, Steps(list_items, screen, read, extract) + source(config=config), browser, client, Steps(list_items, screen, read, extract) ) sites.run(run) @@ -303,7 +305,7 @@ def test_historical_http_failure_does_not_return_partial_backfill(status: int) - with pytest.raises(StepError, match=f"HTTP {status}") as caught: sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime( 2026, 11, 7, tzinfo=timezone(timedelta(hours=8)) @@ -332,7 +334,7 @@ def test_cyclic_pagination_finishes_backfill_without_discarding_items() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -371,7 +373,7 @@ def test_empty_historical_page_finishes_backfill_without_discarding_items() -> N ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -451,7 +453,7 @@ def test_page_parse_failure_is_not_reported_as_a_configuration_error() -> None: } ) with pytest.raises(StepError) as caught: - sites.run(lambda f: list_items(f, source())) + sites.run(lambda f: list_items(PageSession(f, None), source())) assert caught.value.detail == "列表页无法解析" diff --git a/crawler/uv.lock b/crawler/uv.lock index fbf61b9..6ecf832 100644 --- a/crawler/uv.lock +++ b/crawler/uv.lock @@ -112,6 +112,7 @@ source = { editable = "." } dependencies = [ { name = "httpx" }, { name = "lxml" }, + { name = "playwright" }, { name = "pydantic" }, ] @@ -128,6 +129,7 @@ dev = [ requires-dist = [ { name = "httpx", specifier = ">=0.28" }, { name = "lxml", specifier = ">=6.1.3" }, + { name = "playwright", specifier = ">=1.64.1" }, { name = "pydantic", specifier = ">=2.14.1" }, ] @@ -178,6 +180,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/02/767f744ab6d4cb7761e5008acc3d534b7a0481af62563d52e391fbcb2140/genson-1.4.0-py3-none-any.whl", hash = "sha256:03bc71bbe52defde70660cc4dcd1ea1097997da5a1cbb90a9dbd3acc7c9e1b65", size = 24484, upload-time = "2026-07-06T08:21:49.046Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" }, + { url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" }, + { url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" }, + { url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" }, + { url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" }, + { url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" }, + { url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" }, + { url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" }, + { url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" }, + { url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" }, + { url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" }, + { url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" }, + { url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" }, + { url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" }, + { url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" }, + { url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" }, + { url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" }, + { url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" }, + { url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" }, + { url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" }, + { url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -486,6 +535,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, ] +[[package]] +name = "playwright" +version = "1.64.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/c8/cc677856d89a0c077673b40ab915ed5597bf1347b205cabe70121c431ca9/playwright-1.64.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:29453ff76e25c17d0b45658cacd6b92e9eec46e8db21e134fb3b7143df50f013", size = 44508971, upload-time = "2026-10-22T16:29:58.12Z" }, + { url = "https://files.pythonhosted.org/packages/72/4f/572bbbe60a5787ca61a84bad934c3ce19c08c2bc9c0e9781bf0d99cad1a6/playwright-1.64.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:59f7b041284ce8d1b72098ae50ca48a2d83692eec25154ab5ba79d4b6b4fe8a9", size = 43155232, upload-time = "2026-10-22T16:30:02.415Z" }, + { url = "https://files.pythonhosted.org/packages/44/fa/468f313a56d33a6420705147c8d47bef558f087856edf38c5b13eef8aa6c/playwright-1.64.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:6a6f99adad1ae40e6ee62efe89785db861ce7ddd57651521cb31894183cc4ae3", size = 44508977, upload-time = "2026-10-22T16:30:07.632Z" }, + { url = "https://files.pythonhosted.org/packages/15/b9/e481a34572933b5cf179d5637c98da488c732ee339dff5b73a0624d2e42b/playwright-1.64.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:577b8d88778c245cfbd145ae17cf53044f084b1b46c55c69081d4069d56057e2", size = 48471931, upload-time = "2026-10-22T16:30:12.831Z" }, + { url = "https://files.pythonhosted.org/packages/0c/11/545f25e47ec54e1e8a2892ef9994fc2b298f909ab37b88578e3a1f87ac2e/playwright-1.64.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a29fda44725e11b0b625c4bff6458a1f6dbee97b88c9788d12e18a8344458c2d", size = 48150994, upload-time = "2026-10-22T16:30:18.297Z" }, + { url = "https://files.pythonhosted.org/packages/33/87/8ff7af99387ee72ae9b46a66ece7a0c0aa63f73320ecf6dd3dd087fa840a/playwright-1.64.1-py3-none-win32.whl", hash = "sha256:d3c48b31da426c2f3562d0919cb9ea4e5415f6dbd29d7211fa4d66cca24fbd2f", size = 38801828, upload-time = "2026-10-22T16:30:23.154Z" }, + { url = "https://files.pythonhosted.org/packages/96/0a/64db6851aa1852a4f76be28eea71d4e2bf9f3ce493ee2e66939426729046/playwright-1.64.1-py3-none-win_amd64.whl", hash = "sha256:5e240a79464434c9260957b9ef59a5808e02e96e3fcabdc717fd7485260b9730", size = 38801836, upload-time = "2026-10-22T16:30:28.092Z" }, + { url = "https://files.pythonhosted.org/packages/c5/4b/d7ddde9467a5fcfe6a0d3c954732332e98a4f4a9025fd42c8d091c16e5f9/playwright-1.64.1-py3-none-win_arm64.whl", hash = "sha256:983ccd085453ea5a7588193657fc349a6ffbf7bf7ba663fd58dc81e4aa3af9d9", size = 34693875, upload-time = "2026-10-22T16:30:32.724Z" }, +] + [[package]] name = "pluggy" version = "1.7.0" @@ -582,6 +650,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/56/8a702c27e5be9f47e5f19d8669227424290e4c024e7c370279cbaf244b4e/pydantic_core-2.50.1-cp315-cp315t-win_arm64.whl", hash = "sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44", size = 2175297, upload-time = "2026-10-11T18:35:04.079Z" }, ] +[[package]] +name = "pyee" +version = "14.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/fdedc2c75c3e31a330659c85e5793bb18b3397981fbf0844c6dee5b18926/pyee-14.0.0.tar.gz", hash = "sha256:76dd0f4314ecd27f02dc73589dea7fd3853f9b6176d8ef9b122860657e3602de", size = 98760, upload-time = "2026-08-13T04:26:11.021Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/12/5347938b1f9a6453f0dbdfcc3e2388a1320ef9b9ec17fbefbc4ab647ea98/pyee-14.0.0-py3-none-any.whl", hash = "sha256:3ac2d3229a9677f7de2c33d7f52fe25b638a46b19c413fea2edc8c6d0a644e4d", size = 15553, upload-time = "2026-08-13T04:26:09.916Z" }, +] + [[package]] name = "pygments" version = "2.21.0" diff --git a/docs/development.md b/docs/development.md index 794674c..62c08e3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -59,11 +59,9 @@ crawler/ 抓取节点 - 接入来源时按通用抓取 → 配置 → 浏览器抓取 → 专用适配器的顺序试，前一种能抓就不用后一种（[总体设计 §7.2](design.md#72-怎么抓)）。 - 建站系统的模板放在代码里，和它的测试在一起；单个来源的配置存在网站的数据库里，在管理页面改，抓取节点领活时一起拿到。 -- 整个进程共用一个 `Fetcher`（`crawl/fetch.py`），所有来源请求都经过它，遵守[总体设计 §7.3](design.md#73-频率和礼貌)的 UA 和间隔规则。普通 HTTP 请求和附件下载的每次跳转，都按目标网站的间隔排队。 -- 浏览器一次页面访问取一次许可，持有到页面关闭。页面子资源正常加载，跨网站跳转沿用起始网站的许可。 +- 所有对来源网站的请求都经过 `crawl/fetch.py` 的 `Fetcher`，整个进程共用一个，[总体设计 §7.3](design.md#73-频率和礼貌)的规矩由它保证：UA 写明鼠钟；同一个网站一次一个请求，多个任务同时抓也一样；跳转的每一步单独发，跳到别的网站按那个网站的间隔排队。浏览器一次页面访问取一次许可，持有到页面关闭；页面的子资源正常加载，跨网站跳转沿用起始网站的许可。 - 浏览器抓取启动时关掉 Chromium 的自动化标记（`--disable-blink-features=AutomationControlled`），UA 用浏览器自己的，把 `HeadlessChrome` 写成 `Chrome`，末尾加上鼠钟的标识；瑞数这类脚本挑战两处都检查（[总体设计 §5.2](design.md#52-实测)）。不冒充别的浏览器和系统。 -- 每个浏览器来源每轮新建一个无痕会话，列表、正文和附件共用其中的 Cookie，抓完就关。 -- 浏览器同时最多开 2 个页面，多的排队。拒绝弹窗和浏览器下载，附件由程序另外下载；不加载图片、字体和音视频，省对方的流量，也省内存。 +- 一轮里同一个网站共用一个无痕会话，抓完就关：脚本挑战设的 Cookie 在会话里有效，后面的页面不用再过挑战；不同网站、不同轮次之间不共享 Cookie。同时开的浏览器页面有上限，多的排队；拒绝弹窗和下载，附件由程序另外下载；不加载图片、字体和音视频，省对方的流量，也省内存。 - 解析和模型分工：`parse` 用程序取出能确定的东西，模型只做判断（[总体设计 §8.2](design.md#82-抽取)）。解析不猜：判断不了的交给模型或人，不硬配。 - 判别模型和 LLM 的提示词放在调用它的代码旁边。输出都按固定的结构解析，解析不了的当失败处理。 - 每一轮抓取都交一份抓取记录，失败也交，写明原因；不在一轮里反复重试。各步抓不成时抛 `FetchError`（`Fetcher` 抛的）或 `StepError`（写明失败类别，比如 `http_status` 和「HTTP 403」），一轮的流程把它记进抓取记录；别的异常是 bug，照常抛出。 From 3098223de68be23836790024e79c7df94f5dd20e Mon Sep 17 00:00:00 2001 From: lym12321 <41183806+lym12321@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:11:39 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=9C=A8=E6=8A=93=E5=8F=96=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=90=AF=E5=8A=A8=E6=97=B6=E5=88=9D=E5=A7=8B=E5=8C=96?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/src/crawler/crawl/browser.py | 83 ++++++++--------------- crawler/src/crawler/crawl/browser_test.py | 33 +++++++-- crawler/src/crawler/crawl/rounds.py | 4 +- crawler/src/crawler/crawl/rounds_test.py | 30 ++------ crawler/src/crawler/parse/listing_test.py | 12 ++-- 5 files changed, 69 insertions(+), 93 deletions(-) diff --git a/crawler/src/crawler/crawl/browser.py b/crawler/src/crawler/crawl/browser.py index 03aad38..833bb2f 100644 --- a/crawler/src/crawler/crawl/browser.py +++ b/crawler/src/crawler/crawl/browser.py @@ -6,13 +6,9 @@ from typing import Self from urllib.parse import urljoin -from playwright.async_api import ( - Browser as PlaywrightBrowser, -) from playwright.async_api import ( BrowserContext, Page, - Playwright, Route, async_playwright, ) @@ -41,7 +37,7 @@ _REDIRECTS = {301, 302, 303, 307, 308} _BLOCKED = {"image", "font", "media"} _CHALLENGES = {202, 412} -_logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) async def _close_popup(page: Page) -> None: @@ -60,61 +56,38 @@ class Browser: def __init__(self, fetcher: Fetcher) -> None: self.fetcher = fetcher self.slots = asyncio.Semaphore(PAGE_LIMIT) - self._lock = asyncio.Lock() - self._playwright: Playwright | None = None - self._browser: PlaywrightBrowser | None = None - self._user_agent = "" async def __aenter__(self) -> Self: + self._playwright = await async_playwright().start() + try: + self._browser = await self._playwright.chromium.launch( + headless=True, args=["--disable-blink-features=AutomationControlled"] + ) + async with await self._browser.new_page() as page: + user_agent: str = await page.evaluate("navigator.userAgent") + except BaseException: + await self._playwright.stop() + raise + self._user_agent = user_agent.replace("HeadlessChrome", "Chrome") + " " + USER_AGENT return self async def __aexit__(self, *_: object) -> None: - await self.close() - - async def _start(self) -> PlaywrightBrowser: - async with self._lock: - if self._browser is not None: - return self._browser - playwright = await async_playwright().start() - try: - browser = await playwright.chromium.launch( - headless=True, args=["--disable-blink-features=AutomationControlled"] - ) - async with await browser.new_page() as page: - user_agent: str = await page.evaluate("navigator.userAgent") - except BaseException: - await playwright.stop() - raise - self._user_agent = user_agent.replace("HeadlessChrome", "Chrome") + " " + USER_AGENT - self._playwright = playwright - self._browser = browser - return browser + try: + await self._browser.close() + finally: + await self._playwright.stop() @asynccontextmanager async def session(self, source: WorkSource) -> AsyncGenerator[PageSession]: if not source.browser: yield PageSession(self.fetcher, None) return - try: - browser = await self._start() - async with await browser.new_context( - user_agent=self._user_agent, service_workers="block", accept_downloads=False - ) as context: - await context.route("**/*", _route) - context.on("page", _close_popup) - yield PageSession(self.fetcher, BrowserSession(self, context)) - except PlaywrightTimeoutError as error: - raise FetchError("timeout", source.url) from error - except PlaywrightError as error: - raise FetchError("unreachable", source.url) from error - - async def close(self) -> None: - try: - if self._browser is not None: - await self._browser.close() - finally: - if self._playwright is not None: - await self._playwright.stop() + async with await self._browser.new_context( + user_agent=self._user_agent, service_workers="block", accept_downloads=False + ) as context: + await context.route("**/*", _route) + context.on("page", _close_popup) + yield PageSession(self.fetcher, BrowserSession(self, context)) class BrowserSession: @@ -146,11 +119,13 @@ def record_response(response: PlaywrightResponse) -> None: "text/html; charset=utf-8", body.encode("utf-8"), ) - _logger.info( - "Browser fetched %s status=%s elapsed=%.3fs", - response.url, - response.status, - time.monotonic() - started, + logger.info( + "Browser fetched", + extra={ + "url": response.url, + "status": response.status, + "elapsed": time.monotonic() - started, + }, ) return response except PlaywrightTimeoutError as error: diff --git a/crawler/src/crawler/crawl/browser_test.py b/crawler/src/crawler/crawl/browser_test.py index 682c3c8..644a5fb 100644 --- a/crawler/src/crawler/crawl/browser_test.py +++ b/crawler/src/crawler/crawl/browser_test.py @@ -76,6 +76,9 @@ async def goto(self, url: str, *, wait_until: str, **_options: object) -> None: async def content(self) -> str: return "保研通知" + async def evaluate(self, _expression: str) -> str: + return "HeadlessChrome/1.0" + async def __aenter__(self) -> Self: return self @@ -139,26 +142,44 @@ async def __aexit__(self, *_: object) -> None: class FakeBrowser: def __init__(self) -> None: + self.running = False self.contexts: list[FakeContext] = [] self.started: asyncio.Queue[FakePage] = asyncio.Queue() self.block_pages = False self.destination: str | None = None self.responses = [(200, True, True)] + async def launch(self, **_options: object) -> Self: + self.running = True + return self + + async def new_page(self) -> FakePage: + return FakePage(FakeContext(self, 0)) + async def new_context(self, **_options: object) -> FakeContext: context = FakeContext(self, len(self.contexts) + 1) self.contexts.append(context) return context + async def close(self) -> None: + self.running = False + + +class FakePlaywright: + def __init__(self, browser: FakeBrowser) -> None: + self.chromium = browser + + async def start(self) -> Self: + return self + + async def stop(self) -> None: + pass + @pytest.fixture def driver(monkeypatch: pytest.MonkeyPatch) -> FakeBrowser: browser = FakeBrowser() - - async def start(_owner: object) -> FakeBrowser: - return browser - - monkeypatch.setattr("crawler.crawl.browser.Browser._start", start) + monkeypatch.setattr("crawler.crawl.browser.async_playwright", lambda: FakePlaywright(browser)) return browser @@ -242,6 +263,7 @@ async def main() -> None: Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher, Browser(fetcher) as browser, ): + assert driver.running async with browser.session(SOURCE.model_copy(update={"browser": False})): assert driver.contexts == [] async with browser.session(SOURCE) as session: @@ -254,6 +276,7 @@ async def main() -> None: async with browser.session(SOURCE): assert len(driver.contexts) == 2 assert driver.contexts[1].closed + assert not driver.running run(main) diff --git a/crawler/src/crawler/crawl/rounds.py b/crawler/src/crawler/crawl/rounds.py index 8ea00e5..0e55045 100644 --- a/crawler/src/crawler/crawl/rounds.py +++ b/crawler/src/crawler/crawl/rounds.py @@ -5,7 +5,6 @@ import logging import time from collections.abc import Awaitable, Callable -from contextlib import AsyncExitStack from dataclasses import dataclass from crawler.crawl.browser import Browser, PageSession @@ -87,9 +86,8 @@ async def crawl_source( clock: Callable[[], float] = time.time, ) -> None: started_at = int(clock()) - async with AsyncExitStack() as stack: + async with browser.session(source) as pages: try: - pages = await stack.enter_async_context(browser.session(source)) items = await steps.list_items(pages, source) if not items: raise StepError("no_items") diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index 4f54249..f1a2f2a 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -4,7 +4,6 @@ from typing import Any import httpx -import pytest from crawler.crawl.browser import Browser, PageSession from crawler.crawl.contract import ( @@ -65,15 +64,16 @@ def handle(self, request: httpx.Request) -> httpx.Response: ) return httpx.Response(204) - def run(self, steps: Steps, source: WorkSource = SOURCE) -> None: + def run(self, steps: Steps) -> None: async def main() -> None: nowhere = httpx.MockTransport(lambda _: httpx.Response(500)) async with ( Fetcher(nowhere) as fetcher, - Browser(fetcher) as browser, Website("http://site", "token", httpx.MockTransport(self.handle)) as website, ): - await crawl_source(source, browser, website, steps, clock=lambda: 1_800_000_000) + await crawl_source( + SOURCE, Browser(fetcher), website, steps, clock=lambda: 1_800_000_000 + ) asyncio.run(main()) @@ -212,25 +212,3 @@ async def unused(*_: object) -> Any: [run] = site.called("POST /runs") assert run["failure"] == failure assert run["listed"] == 0 - - -def test_a_session_that_cant_start_is_reported(monkeypatch: pytest.MonkeyPatch) -> None: - site = FakeWebsite([], extracted=frozenset()) - - async def unavailable(_browser: Browser) -> None: - raise FetchError("unreachable", SOURCE.url) - - async def unused_read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: - raise AssertionError("Listing failed before notices could be read") - - monkeypatch.setattr(Browser, "_start", unavailable) - site.run( - Steps(list_items, screen, unused_read, extract), - SOURCE.model_copy(update={"browser": True}), - ) - - assert [call for call, _ in site.calls] == ["POST /runs"] - [run] = site.called("POST /runs") - assert run["failure"] == "unreachable" - assert run["detail"] == SOURCE.url - assert run["listed"] == 0 diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py index a37ce50..9c405f6 100644 --- a/crawler/src/crawler/parse/listing_test.py +++ b/crawler/src/crawler/parse/listing_test.py @@ -279,12 +279,14 @@ async def read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePag raise AssertionError("a failed list must not read notices") async def run(fetcher: Fetcher) -> None: - async with ( - Browser(fetcher) as browser, - Website("http://site", "local-test", httpx.MockTransport(website.handle)) as client, - ): + async with Website( + "http://site", "local-test", httpx.MockTransport(website.handle) + ) as client: await crawl_source( - source(config=config), browser, client, Steps(list_items, screen, read, extract) + source(config=config), + Browser(fetcher), + client, + Steps(list_items, screen, read, extract), ) sites.run(run)