diff --git a/crawler/pyproject.toml b/crawler/pyproject.toml index c640430..ac2d3b4 100644 --- a/crawler/pyproject.toml +++ b/crawler/pyproject.toml @@ -6,6 +6,7 @@ requires-python = ">=3.14" dependencies = [ "httpx>=0.28", "lxml>=6.1.3", + "playwright>=1.64.1", "pydantic>=2.14.1", ] diff --git a/crawler/src/crawler/crawl/browser.py b/crawler/src/crawler/crawl/browser.py new file mode 100644 index 0000000..833bb2f --- /dev/null +++ b/crawler/src/crawler/crawl/browser.py @@ -0,0 +1,181 @@ +import asyncio +import logging +import time +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from typing import Self +from urllib.parse import urljoin + +from playwright.async_api import ( + BrowserContext, + Page, + Route, + async_playwright, +) +from playwright.async_api import ( + Error as PlaywrightError, +) +from playwright.async_api import ( + Response as PlaywrightResponse, +) +from playwright.async_api import ( + TimeoutError as PlaywrightTimeoutError, +) + +from crawler.crawl.contract import WorkSource +from crawler.crawl.fetch import ( + MAX_REDIRECTS, + TIMEOUT, + USER_AGENT, + Fetcher, + FetchError, + Response, +) + +PAGE_TIMEOUT = 300.0 +PAGE_LIMIT = 2 +_REDIRECTS = {301, 302, 303, 307, 308} +_BLOCKED = {"image", "font", "media"} +_CHALLENGES = {202, 412} +logger = logging.getLogger(__name__) + + +async def _close_popup(page: Page) -> None: + if await page.opener() is not None: + await page.close() + + +async def _route(route: Route) -> None: + if route.request.resource_type in _BLOCKED: + await route.abort() + else: + await route.continue_() + + +class Browser: + def __init__(self, fetcher: Fetcher) -> None: + self.fetcher = fetcher + self.slots = asyncio.Semaphore(PAGE_LIMIT) + + async def __aenter__(self) -> Self: + self._playwright = await async_playwright().start() + try: + self._browser = await self._playwright.chromium.launch( + headless=True, args=["--disable-blink-features=AutomationControlled"] + ) + async with await self._browser.new_page() as page: + user_agent: str = await page.evaluate("navigator.userAgent") + except BaseException: + await self._playwright.stop() + raise + self._user_agent = user_agent.replace("HeadlessChrome", "Chrome") + " " + USER_AGENT + return self + + async def __aexit__(self, *_: object) -> None: + try: + await self._browser.close() + finally: + await self._playwright.stop() + + @asynccontextmanager + async def session(self, source: WorkSource) -> AsyncGenerator[PageSession]: + if not source.browser: + yield PageSession(self.fetcher, None) + return + async with await self._browser.new_context( + user_agent=self._user_agent, service_workers="block", accept_downloads=False + ) as context: + await context.route("**/*", _route) + context.on("page", _close_popup) + yield PageSession(self.fetcher, BrowserSession(self, context)) + + +class BrowserSession: + def __init__(self, owner: Browser, context: BrowserContext) -> None: + self._owner = owner + self._context = context + + async def get(self, url: str) -> Response: + async with self._owner.slots, self._owner.fetcher.permit(url): + started = time.monotonic() + try: + async with await self._context.new_page() as page: + last_response: PlaywrightResponse | None = None + + def record_response(response: PlaywrightResponse) -> None: + nonlocal last_response + request = response.request + if request.is_navigation_request() and request.frame == page.main_frame: + last_response = response + + page.on("response", record_response) + await page.goto(url, wait_until="networkidle", timeout=PAGE_TIMEOUT * 1000) + if last_response is None or last_response.status in _CHALLENGES: + raise FetchError("unreachable", page.url) + body = await page.content() + response = Response( + page.url, + last_response.status, + "text/html; charset=utf-8", + body.encode("utf-8"), + ) + logger.info( + "Browser fetched", + extra={ + "url": response.url, + "status": response.status, + "elapsed": time.monotonic() - started, + }, + ) + return response + except PlaywrightTimeoutError as error: + raise FetchError("timeout", url) from error + except PlaywrightError as error: + raise FetchError("unreachable", url) from error + + async def download(self, url: str, *, page_url: str) -> Response: + for _ in range(MAX_REDIRECTS + 1): + async with self._owner.fetcher.permit(url): + try: + response = await self._context.request.get( + url, + headers={"Referer": page_url}, + timeout=TIMEOUT * 1000, + max_redirects=0, + max_retries=0, + fail_on_status_code=False, + ) + try: + location = response.headers.get("location") + if response.status in _REDIRECTS and location is not None: + url = urljoin(response.url, location) + continue + return Response( + response.url, + response.status, + response.headers.get("content-type"), + await response.body(), + ) + finally: + await response.dispose() + except PlaywrightTimeoutError as error: + raise FetchError("timeout", url) from error + except PlaywrightError as error: + raise FetchError("unreachable", url) from error + raise FetchError("too_many_redirects", url) + + +class PageSession: + def __init__(self, fetcher: Fetcher, browser_session: BrowserSession | None) -> None: + self._fetcher = fetcher + self._browser_session = browser_session + + async def get(self, url: str) -> Response: + if self._browser_session is not None: + return await self._browser_session.get(url) + return await self._fetcher.get(url) + + async def download(self, url: str, *, page_url: str) -> Response: + if self._browser_session is not None: + return await self._browser_session.download(url, page_url=page_url) + return await self._fetcher.get(url) diff --git a/crawler/src/crawler/crawl/browser_test.py b/crawler/src/crawler/crawl/browser_test.py new file mode 100644 index 0000000..644a5fb --- /dev/null +++ b/crawler/src/crawler/crawl/browser_test.py @@ -0,0 +1,346 @@ +import asyncio +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Self + +import httpx +import pytest + +from crawler.crawl.browser import Browser +from crawler.crawl.contract import WorkSource +from crawler.crawl.fetch import INTERVAL, Fetcher, FetchError + +SOURCE = WorkSource( + id=7, url="https://a/list", section="通知公告", browser=True, config=None, backfill=False +) + + +def run(main: Callable[[], Awaitable[None]]) -> None: + async def bounded() -> None: + async with asyncio.timeout(5): + await main() + + asyncio.run(bounded()) + + +class FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def clock(self) -> float: + return self.now + + async def sleep(self, seconds: float) -> None: + self.now += seconds + await asyncio.sleep(0) + + +@dataclass +class FakeRequest: + frame: object + navigation: bool + + def is_navigation_request(self) -> bool: + return self.navigation + + +@dataclass +class FakeResponse: + status: int + request: FakeRequest + + +class FakePage: + def __init__(self, context: FakeContext) -> None: + self.context = context + self.main_frame = object() + self.url = "about:blank" + self.closed = False + self.ready = asyncio.Event() + self.handlers: dict[str, Callable[[FakeResponse], None]] = {} + + def on(self, event: str, callback: Callable[[FakeResponse], None]) -> None: + self.handlers[event] = callback + + async def goto(self, url: str, *, wait_until: str, **_options: object) -> None: + assert wait_until == "networkidle" + driver = self.context.driver + self.url = driver.destination or url + driver.started.put_nowait(self) + if driver.block_pages: + await self.ready.wait() + for status, navigation, main_frame in driver.responses: + frame = self.main_frame if main_frame else object() + self.handlers["response"](FakeResponse(status, FakeRequest(frame, navigation))) + + async def content(self) -> str: + return "保研通知" + + async def evaluate(self, _expression: str) -> str: + return "HeadlessChrome/1.0" + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *_: object) -> None: + self.closed = True + + +class FakeDownload: + def __init__(self, url: str, body: bytes) -> None: + self.url = url + self.status = 200 + self.headers = {"content-type": "application/pdf"} + self.data = body + self.disposed = False + + async def body(self) -> bytes: + return self.data + + async def dispose(self) -> None: + self.disposed = True + + +class FakeDownloads: + def __init__(self, context: FakeContext) -> None: + self.context = context + self.responses: list[FakeDownload] = [] + + async def get(self, url: str, *, headers: dict[str, str], **_options: object) -> FakeDownload: + response = FakeDownload( + url, f"%PDF context {self.context.number}: {headers['Referer']}".encode() + ) + self.responses.append(response) + return response + + +class FakeContext: + def __init__(self, driver: FakeBrowser, number: int) -> None: + self.driver = driver + self.number = number + self.request = FakeDownloads(self) + self.pages: list[FakePage] = [] + self.closed = False + + async def route(self, _pattern: str, _handler: object) -> None: + pass + + def on(self, _event: str, _handler: object) -> None: + pass + + async def new_page(self) -> FakePage: + page = FakePage(self) + self.pages.append(page) + return page + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *_: object) -> None: + self.closed = True + + +class FakeBrowser: + def __init__(self) -> None: + self.running = False + self.contexts: list[FakeContext] = [] + self.started: asyncio.Queue[FakePage] = asyncio.Queue() + self.block_pages = False + self.destination: str | None = None + self.responses = [(200, True, True)] + + async def launch(self, **_options: object) -> Self: + self.running = True + return self + + async def new_page(self) -> FakePage: + return FakePage(FakeContext(self, 0)) + + async def new_context(self, **_options: object) -> FakeContext: + context = FakeContext(self, len(self.contexts) + 1) + self.contexts.append(context) + return context + + async def close(self) -> None: + self.running = False + + +class FakePlaywright: + def __init__(self, browser: FakeBrowser) -> None: + self.chromium = browser + + async def start(self) -> Self: + return self + + async def stop(self) -> None: + pass + + +@pytest.fixture +def driver(monkeypatch: pytest.MonkeyPatch) -> FakeBrowser: + browser = FakeBrowser() + monkeypatch.setattr("crawler.crawl.browser.async_playwright", lambda: FakePlaywright(browser)) + return browser + + +def test_page_visits_and_http_requests_wait_for_each_other(driver: FakeBrowser) -> None: + driver.block_pages = True + clock = FakeClock() + http_started = asyncio.Event() + finish_http = asyncio.Event() + request_times: list[float] = [] + + async def handle(_request: httpx.Request) -> httpx.Response: + request_times.append(clock.now) + http_started.set() + await finish_http.wait() + return httpx.Response(200) + + async def main() -> None: + async with ( + Fetcher(httpx.MockTransport(handle), clock.clock, clock.sleep) as fetcher, + Browser(fetcher) as browser, + browser.session(SOURCE) as session, + ): + first_http = asyncio.create_task(fetcher.get("https://a/first")) + await http_started.wait() + visit = asyncio.create_task(session.get("https://a/list")) + await asyncio.sleep(0) + assert driver.started.empty() + clock.now = 2 + finish_http.set() + await first_http + page = await driver.started.get() + assert clock.now == 2 + INTERVAL + second_http = asyncio.create_task(fetcher.get("https://a/second")) + await asyncio.sleep(0) + assert request_times == [0] + clock.now = 10 + page.ready.set() + await visit + await second_http + assert page.closed + assert request_times == [0, 10 + INTERVAL] + + run(main) + + +def test_page_limit_is_shared_by_rounds(driver: FakeBrowser) -> None: + driver.block_pages = True + + async def main() -> None: + async with ( + Fetcher() as fetcher, + Browser(fetcher) as browser, + browser.session(SOURCE) as first, + browser.session(SOURCE.model_copy(update={"id": 8, "url": "https://b/list"})) as second, + ): + tasks = [ + asyncio.create_task(first.get("https://a/list")), + asyncio.create_task(second.get("https://b/list")), + asyncio.create_task(first.get("https://c/list")), + ] + first_page = await driver.started.get() + second_page = await driver.started.get() + await asyncio.sleep(0) + assert sum(len(context.pages) for context in driver.contexts) == 2 + first_page.ready.set() + third_page = await driver.started.get() + assert first_page.closed + second_page.ready.set() + third_page.ready.set() + await asyncio.gather(*tasks) + assert all(page.closed for context in driver.contexts for page in context.pages) + + run(main) + + +def test_each_round_opens_reuses_and_closes_its_context(driver: FakeBrowser) -> None: + clock = FakeClock() + + async def main() -> None: + async with ( + Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher, + Browser(fetcher) as browser, + ): + assert driver.running + async with browser.session(SOURCE.model_copy(update={"browser": False})): + assert driver.contexts == [] + async with browser.session(SOURCE) as session: + assert len(driver.contexts) == 1 + await session.get("https://a/list") + await session.get("https://a/notice") + assert len(driver.contexts[0].pages) == 2 + assert not driver.contexts[0].closed + assert driver.contexts[0].closed + async with browser.session(SOURCE): + assert len(driver.contexts) == 2 + assert driver.contexts[1].closed + assert not driver.running + + run(main) + + +def test_downloads_use_the_rounds_context_request_client(driver: FakeBrowser) -> None: + clock = FakeClock() + + async def main() -> None: + async with ( + Fetcher(clock=clock.clock, sleep=clock.sleep) as fetcher, + Browser(fetcher) as browser, + ): + for number in (1, 2): + async with browser.session(SOURCE) as session: + await session.get("https://a/notice") + attachment = await session.download( + "https://a/file.pdf", page_url="https://a/notice" + ) + assert attachment.body == f"%PDF context {number}: https://a/notice".encode() + assert attachment.content_type == "application/pdf" + assert driver.contexts[number - 1].request.responses[0].disposed + + run(main) + + +def test_last_main_navigation_response_is_used_without_a_cross_site_permit( + driver: FakeBrowser, +) -> None: + driver.destination = "https://b/notice" + driver.responses = [ + (202, True, True), + (200, True, True), + (404, False, True), + (500, True, False), + ] + + async def main() -> None: + async with ( + Fetcher() as fetcher, + Browser(fetcher) as browser, + fetcher.permit("https://b/busy"), + browser.session(SOURCE) as session, + ): + response = await session.get("https://a/notice") + assert response.status == 200 + assert response.url == "https://b/notice" + assert response.body.decode() == "保研通知" + + run(main) + + +@pytest.mark.parametrize("status", [202, 412]) +def test_unresolved_challenges_fail(driver: FakeBrowser, status: int) -> None: + driver.responses = [(status, True, True)] + + async def main() -> None: + async with ( + Fetcher() as fetcher, + Browser(fetcher) as browser, + browser.session(SOURCE) as session, + ): + with pytest.raises(FetchError) as caught: + await session.get("https://a/list") + assert caught.value.failure == "unreachable" + assert driver.contexts[0].pages[0].closed + + run(main) diff --git a/crawler/src/crawler/crawl/fetch.py b/crawler/src/crawler/crawl/fetch.py index 35a43b4..742c017 100644 --- a/crawler/src/crawler/crawl/fetch.py +++ b/crawler/src/crawler/crawl/fetch.py @@ -1,18 +1,17 @@ -"""Every request to a source site goes through here, so the crawl keeps its manners in one place -(design §7.3): it says who it is and sends one request at a time to a site with a pause between -them.""" +"""Source access shares one policy here (design §7.3): identify the crawler and allow one +page visit or HTTP request at a time per site, with a pause between them.""" import asyncio import time -from collections.abc import Awaitable, Callable +from collections.abc import AsyncGenerator, Awaitable, Callable +from contextlib import asynccontextmanager from dataclasses import dataclass, field from typing import Literal, Self -from urllib.parse import urlsplit import httpx USER_AGENT = "mouseclock/0.1 (+https://github.com/mousebaoyan/mouseclock)" -# Seconds from the end of one request to a site to the start of the next. +# Seconds from the end of one page visit or HTTP request to the start of the next. INTERVAL = 5.0 MAX_REDIRECTS = 5 TIMEOUT = 30.0 @@ -63,6 +62,18 @@ async def __aenter__(self) -> Self: async def __aexit__(self, *_: object) -> None: await self._client.aclose() + @asynccontextmanager + async def permit(self, url: str) -> AsyncGenerator[None]: + site = self._sites.setdefault(httpx.URL(url).netloc.decode("ascii"), _Site()) + async with site.lock: + wait = site.finished_at + INTERVAL - self._clock() + if wait > 0: + await self._sleep(wait) + try: + yield + finally: + site.finished_at = self._clock() + async def get(self, url: str) -> Response: # Each hop on its own, so a redirect to another site waits its turn there too. for _ in range(MAX_REDIRECTS + 1): @@ -78,16 +89,10 @@ async def get(self, url: str) -> Response: raise FetchError("too_many_redirects", url) async def _send(self, url: str) -> httpx.Response: - site = self._sites.setdefault(urlsplit(url).netloc, _Site()) - async with site.lock: - wait = site.finished_at + INTERVAL - self._clock() - if wait > 0: - await self._sleep(wait) + async with self.permit(url): try: return await self._client.get(url) except httpx.TimeoutException as error: raise FetchError("timeout", url) from error except httpx.TransportError as error: raise FetchError("unreachable", url) from error - finally: - site.finished_at = self._clock() diff --git a/crawler/src/crawler/crawl/listing.py b/crawler/src/crawler/crawl/listing.py index 4eb1bdb..0c51853 100644 --- a/crawler/src/crawler/crawl/listing.py +++ b/crawler/src/crawler/crawl/listing.py @@ -6,14 +6,14 @@ from pydantic import ValidationError +from crawler.crawl.browser import PageSession from crawler.crawl.contract import ListedItem, WorkSource -from crawler.crawl.fetch import Fetcher from crawler.crawl.rounds import StepError from crawler.parse.listing import ListConfig, parse_listing async def list_items( - fetcher: Fetcher, source: WorkSource, clock: Callable[[], float] = time.time + pages: PageSession, source: WorkSource, clock: Callable[[], float] = time.time ) -> list[ListedItem]: try: config = ListConfig.model_validate(source.config or {}) @@ -37,7 +37,7 @@ def request_url(url: str) -> str: if current in visited: break visited.add(current) - response = await fetcher.get(current) + response = await pages.get(current) visited.add(response.url) if not 200 <= response.status < 300: raise StepError("http_status", f"HTTP {response.status}") diff --git a/crawler/src/crawler/crawl/rounds.py b/crawler/src/crawler/crawl/rounds.py index 78d87d2..0e55045 100644 --- a/crawler/src/crawler/crawl/rounds.py +++ b/crawler/src/crawler/crawl/rounds.py @@ -7,6 +7,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass +from crawler.crawl.browser import Browser, PageSession from crawler.crawl.contract import ( AttachmentInfo, ExtractedProgram, @@ -24,7 +25,7 @@ Usage, WorkSource, ) -from crawler.crawl.fetch import Fetcher, FetchError +from crawler.crawl.fetch import FetchError from crawler.crawl.website import Website logger = logging.getLogger(__name__) @@ -57,14 +58,14 @@ class NoticePage: @dataclass(frozen=True) class Steps: # The items on the source's list page, paging back two years when it is new (design §7.1). - list_items: Callable[[Fetcher, WorkSource], Awaitable[list[ListedItem]]] + list_items: Callable[[PageSession, WorkSource], Awaitable[list[ListedItem]]] # A score for each notice by its title alone (design §8.1). screen: Callable[ [WorkSource, list[PendingNotice]], Awaitable[tuple[list[ScreeningResult], list[Usage]]] ] # A notice's page with its attachments downloaded and read; None when the page answers # 404 or 410 (design §7.4). - read: Callable[[Fetcher, WorkSource, str], Awaitable[NoticePage | None]] + read: Callable[[PageSession, WorkSource, str], Awaitable[NoticePage | None]] # The programs a notice is about (design §8.2). extract: Callable[ [NoticePage, ExtractionContext], Awaitable[tuple[list[ExtractedProgram], list[Usage]]] @@ -79,67 +80,68 @@ def _failure(error: FetchError | StepError) -> tuple[Failure, str | None]: async def crawl_source( source: WorkSource, - fetcher: Fetcher, + browser: Browser, website: Website, steps: Steps, clock: Callable[[], float] = time.time, ) -> None: started_at = int(clock()) - try: - items = await steps.list_items(fetcher, source) - if not items: - raise StepError("no_items") - except (FetchError, StepError) as error: - logger.warning("Listing failed: %s", error, extra={"source_id": source.id}) - failure, detail = _failure(error) + async with browser.session(source) as pages: + try: + items = await steps.list_items(pages, source) + if not items: + raise StepError("no_items") + except (FetchError, StepError) as error: + logger.warning("Listing failed: %s", error, extra={"source_id": source.id}) + failure, detail = _failure(error) + await website.run( + Run( + source_id=source.id, + started_at=started_at, + finished_at=int(clock()), + failure=failure, + detail=detail, + listed=0, + new_count=0, + useful_count=0, + notice_failures=[], + ) + ) + return + + listing = await website.listing(source.id, Listing(items=items)) + new = [notice for notice in listing.pending if notice.step == "screen"] + to_fetch = [notice for notice in listing.pending if notice.step == "fetch"] + if new: + results, usage = await steps.screen(source, new) + await website.screening(source.id, Screening(results=results, usage=usage)) + screened_in = {result.notice_id for result in results if result.screened_in} + to_fetch += [notice for notice in new if notice.id in screened_in] + + useful = 0 + failures: list[NoticeFailure] = [] + for notice in to_fetch: + try: + programs = await _fetch_notice(notice, source, pages, website, steps, clock) + except (FetchError, StepError) as error: + logger.warning("Fetching a notice failed: %s", error, extra={"url": notice.url}) + failure, detail = _failure(error) + failures.append(NoticeFailure(url=notice.url, failure=failure, detail=detail)) + continue + if programs and notice.step == "screen": + useful += 1 + await website.run( Run( source_id=source.id, started_at=started_at, finished_at=int(clock()), - failure=failure, - detail=detail, - listed=0, - new_count=0, - useful_count=0, - notice_failures=[], + listed=len(items), + new_count=listing.new_count, + useful_count=useful, + notice_failures=failures, ) ) - return - - listing = await website.listing(source.id, Listing(items=items)) - new = [notice for notice in listing.pending if notice.step == "screen"] - to_fetch = [notice for notice in listing.pending if notice.step == "fetch"] - if new: - results, usage = await steps.screen(source, new) - await website.screening(source.id, Screening(results=results, usage=usage)) - screened_in = {result.notice_id for result in results if result.screened_in} - to_fetch += [notice for notice in new if notice.id in screened_in] - - useful = 0 - failures: list[NoticeFailure] = [] - for notice in to_fetch: - try: - programs = await _fetch_notice(notice, source, fetcher, website, steps, clock) - except (FetchError, StepError) as error: - logger.warning("Fetching a notice failed: %s", error, extra={"url": notice.url}) - failure, detail = _failure(error) - failures.append(NoticeFailure(url=notice.url, failure=failure, detail=detail)) - continue - if programs and notice.step == "screen": - useful += 1 - - await website.run( - Run( - source_id=source.id, - started_at=started_at, - finished_at=int(clock()), - listed=len(items), - new_count=listing.new_count, - useful_count=useful, - notice_failures=failures, - ) - ) # Returns how many programs the notice turned out to be about; none when it was gone, @@ -147,12 +149,12 @@ async def crawl_source( async def _fetch_notice( notice: PendingNotice, source: WorkSource, - fetcher: Fetcher, + pages: PageSession, website: Website, steps: Steps, clock: Callable[[], float], ) -> int: - page = await steps.read(fetcher, source, notice.url) + page = await steps.read(pages, source, notice.url) if page is None: await website.removed(notice.id) return 0 diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index 0b06365..f1a2f2a 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -5,6 +5,7 @@ import httpx +from crawler.crawl.browser import Browser, PageSession from crawler.crawl.contract import ( AttachmentInfo, ExtractedFields, @@ -70,7 +71,9 @@ async def main() -> None: Fetcher(nowhere) as fetcher, Website("http://site", "token", httpx.MockTransport(self.handle)) as website, ): - await crawl_source(SOURCE, fetcher, website, steps, clock=lambda: 1_800_000_000) + await crawl_source( + SOURCE, Browser(fetcher), website, steps, clock=lambda: 1_800_000_000 + ) asyncio.run(main()) @@ -78,7 +81,7 @@ def called(self, prefix: str) -> list[Any]: return [body for call, body in self.calls if call.startswith(prefix)] -async def list_items(_fetcher: Fetcher, _source: WorkSource) -> list[ListedItem]: +async def list_items(_pages: PageSession, _source: WorkSource) -> list[ListedItem]: return ITEMS @@ -124,7 +127,7 @@ def test_a_round_screens_the_new_fetches_what_needs_it_and_reports() -> None: extracted=frozenset({3}), ) - async def read(_fetcher: Fetcher, _source: WorkSource, url: str) -> NoticePage | None: + async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None: return {"http://a/1.htm": page("冬令营", files=1), "http://a/3.htm": page("夏令营")}.get( url ) @@ -165,32 +168,37 @@ async def read(_fetcher: Fetcher, _source: WorkSource, url: str) -> NoticePage | def test_notices_that_fail_are_reported_and_the_round_goes_on() -> None: - site = FakeWebsite( - [pending(1, "fetch"), pending(2, "fetch"), pending(3, "fetch")], extracted=frozenset() - ) + site = FakeWebsite([pending(n, "fetch") for n in range(1, 5)], extracted=frozenset()) - async def read(_fetcher: Fetcher, _source: WorkSource, url: str) -> NoticePage | None: + async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None: if url == "http://a/1.htm": raise FetchError("timeout", url) if url == "http://a/2.htm": raise StepError("http_status", "HTTP 403") + if url == "http://a/3.htm": + raise FetchError("unreachable", "http://a/files/application.pdf") return page("冬令营") site.run(Steps(list_items, screen, read, extract)) - assert len(site.called("POST /snapshots/30/extraction")) == 1 + assert len(site.called("POST /snapshots/40/extraction")) == 1 [run] = site.called("POST /runs") assert run["notice_failures"] == [ {"url": "http://a/1.htm", "failure": "timeout", "detail": "http://a/1.htm"}, {"url": "http://a/2.htm", "failure": "http_status", "detail": "HTTP 403"}, + { + "url": "http://a/3.htm", + "failure": "unreachable", + "detail": "http://a/files/application.pdf", + }, ] def test_a_list_that_cant_be_read_ends_the_round_with_the_reason() -> None: - async def unreachable(_fetcher: Fetcher, source: WorkSource) -> list[ListedItem]: + async def unreachable(_pages: PageSession, source: WorkSource) -> list[ListedItem]: raise FetchError("unreachable", source.url) - async def empty(_fetcher: Fetcher, _source: WorkSource) -> list[ListedItem]: + async def empty(_pages: PageSession, _source: WorkSource) -> list[ListedItem]: return [] async def unused(*_: object) -> Any: diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py index ba5da48..9c405f6 100644 --- a/crawler/src/crawler/parse/listing_test.py +++ b/crawler/src/crawler/parse/listing_test.py @@ -5,6 +5,7 @@ import pytest from pydantic import JsonValue +from crawler.crawl.browser import Browser, PageSession from crawler.crawl.contract import WorkSource from crawler.crawl.fetch import Fetcher from crawler.crawl.fetch_test import FakeSites @@ -107,7 +108,7 @@ def test_documented_http_configuration_reads_the_real_page_structure() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(config=config), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -144,7 +145,7 @@ def test_backfill_keeps_boundary_and_unknown_dates_and_ignores_repeated_pinned_r ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=backfill), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -168,7 +169,7 @@ def test_leap_day_backfill_uses_calendar_years() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2024, 2, 29, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -199,7 +200,7 @@ def test_numbered_pager_advances_one_page_and_stops_at_last() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -240,7 +241,7 @@ def test_explicit_http_config_and_redirect_base_are_used_for_pagination_and_link config["selectors"] = selectors result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True, config=config), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -274,7 +275,7 @@ def test_bad_configuration_is_recorded_by_the_round( sites = FakeSites({BASE: httpx.Response(200, content=page([("1", "2026-09-10")]))}) website = FakeWebsite([], frozenset()) - async def read(_fetcher: Fetcher, _source: WorkSource, _url: str) -> NoticePage: + async def read(_pages: PageSession, _source: WorkSource, _url: str) -> NoticePage: raise AssertionError("a failed list must not read notices") async def run(fetcher: Fetcher) -> None: @@ -282,7 +283,10 @@ async def run(fetcher: Fetcher) -> None: "http://site", "local-test", httpx.MockTransport(website.handle) ) as client: await crawl_source( - source(config=config), fetcher, client, Steps(list_items, screen, read, extract) + source(config=config), + Browser(fetcher), + client, + Steps(list_items, screen, read, extract), ) sites.run(run) @@ -303,7 +307,7 @@ def test_historical_http_failure_does_not_return_partial_backfill(status: int) - with pytest.raises(StepError, match=f"HTTP {status}") as caught: sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime( 2026, 11, 7, tzinfo=timezone(timedelta(hours=8)) @@ -332,7 +336,7 @@ def test_cyclic_pagination_finishes_backfill_without_discarding_items() -> None: ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -371,7 +375,7 @@ def test_empty_historical_page_finishes_backfill_without_discarding_items() -> N ) result = sites.run( lambda f: list_items( - f, + PageSession(f, None), source(backfill=True), clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) @@ -451,7 +455,7 @@ def test_page_parse_failure_is_not_reported_as_a_configuration_error() -> None: } ) with pytest.raises(StepError) as caught: - sites.run(lambda f: list_items(f, source())) + sites.run(lambda f: list_items(PageSession(f, None), source())) assert caught.value.detail == "列表页无法解析" diff --git a/crawler/uv.lock b/crawler/uv.lock index fbf61b9..6ecf832 100644 --- a/crawler/uv.lock +++ b/crawler/uv.lock @@ -112,6 +112,7 @@ source = { editable = "." } dependencies = [ { name = "httpx" }, { name = "lxml" }, + { name = "playwright" }, { name = "pydantic" }, ] @@ -128,6 +129,7 @@ dev = [ requires-dist = [ { name = "httpx", specifier = ">=0.28" }, { name = "lxml", specifier = ">=6.1.3" }, + { name = "playwright", specifier = ">=1.64.1" }, { name = "pydantic", specifier = ">=2.14.1" }, ] @@ -178,6 +180,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/02/767f744ab6d4cb7761e5008acc3d534b7a0481af62563d52e391fbcb2140/genson-1.4.0-py3-none-any.whl", hash = "sha256:03bc71bbe52defde70660cc4dcd1ea1097997da5a1cbb90a9dbd3acc7c9e1b65", size = 24484, upload-time = "2026-07-06T08:21:49.046Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" }, + { url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" }, + { url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" }, + { url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" }, + { url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" }, + { url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" }, + { url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" }, + { url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" }, + { url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" }, + { url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" }, + { url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" }, + { url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" }, + { url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" }, + { url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" }, + { url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" }, + { url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" }, + { url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" }, + { url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" }, + { url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" }, + { url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" }, + { url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -486,6 +535,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, ] +[[package]] +name = "playwright" +version = "1.64.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/c8/cc677856d89a0c077673b40ab915ed5597bf1347b205cabe70121c431ca9/playwright-1.64.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:29453ff76e25c17d0b45658cacd6b92e9eec46e8db21e134fb3b7143df50f013", size = 44508971, upload-time = "2026-10-22T16:29:58.12Z" }, + { url = "https://files.pythonhosted.org/packages/72/4f/572bbbe60a5787ca61a84bad934c3ce19c08c2bc9c0e9781bf0d99cad1a6/playwright-1.64.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:59f7b041284ce8d1b72098ae50ca48a2d83692eec25154ab5ba79d4b6b4fe8a9", size = 43155232, upload-time = "2026-10-22T16:30:02.415Z" }, + { url = "https://files.pythonhosted.org/packages/44/fa/468f313a56d33a6420705147c8d47bef558f087856edf38c5b13eef8aa6c/playwright-1.64.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:6a6f99adad1ae40e6ee62efe89785db861ce7ddd57651521cb31894183cc4ae3", size = 44508977, upload-time = "2026-10-22T16:30:07.632Z" }, + { url = "https://files.pythonhosted.org/packages/15/b9/e481a34572933b5cf179d5637c98da488c732ee339dff5b73a0624d2e42b/playwright-1.64.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:577b8d88778c245cfbd145ae17cf53044f084b1b46c55c69081d4069d56057e2", size = 48471931, upload-time = "2026-10-22T16:30:12.831Z" }, + { url = "https://files.pythonhosted.org/packages/0c/11/545f25e47ec54e1e8a2892ef9994fc2b298f909ab37b88578e3a1f87ac2e/playwright-1.64.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a29fda44725e11b0b625c4bff6458a1f6dbee97b88c9788d12e18a8344458c2d", size = 48150994, upload-time = "2026-10-22T16:30:18.297Z" }, + { url = "https://files.pythonhosted.org/packages/33/87/8ff7af99387ee72ae9b46a66ece7a0c0aa63f73320ecf6dd3dd087fa840a/playwright-1.64.1-py3-none-win32.whl", hash = "sha256:d3c48b31da426c2f3562d0919cb9ea4e5415f6dbd29d7211fa4d66cca24fbd2f", size = 38801828, upload-time = "2026-10-22T16:30:23.154Z" }, + { url = "https://files.pythonhosted.org/packages/96/0a/64db6851aa1852a4f76be28eea71d4e2bf9f3ce493ee2e66939426729046/playwright-1.64.1-py3-none-win_amd64.whl", hash = "sha256:5e240a79464434c9260957b9ef59a5808e02e96e3fcabdc717fd7485260b9730", size = 38801836, upload-time = "2026-10-22T16:30:28.092Z" }, + { url = "https://files.pythonhosted.org/packages/c5/4b/d7ddde9467a5fcfe6a0d3c954732332e98a4f4a9025fd42c8d091c16e5f9/playwright-1.64.1-py3-none-win_arm64.whl", hash = "sha256:983ccd085453ea5a7588193657fc349a6ffbf7bf7ba663fd58dc81e4aa3af9d9", size = 34693875, upload-time = "2026-10-22T16:30:32.724Z" }, +] + [[package]] name = "pluggy" version = "1.7.0" @@ -582,6 +650,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/56/8a702c27e5be9f47e5f19d8669227424290e4c024e7c370279cbaf244b4e/pydantic_core-2.50.1-cp315-cp315t-win_arm64.whl", hash = "sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44", size = 2175297, upload-time = "2026-10-11T18:35:04.079Z" }, ] +[[package]] +name = "pyee" +version = "14.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/fdedc2c75c3e31a330659c85e5793bb18b3397981fbf0844c6dee5b18926/pyee-14.0.0.tar.gz", hash = "sha256:76dd0f4314ecd27f02dc73589dea7fd3853f9b6176d8ef9b122860657e3602de", size = 98760, upload-time = "2026-08-13T04:26:11.021Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/12/5347938b1f9a6453f0dbdfcc3e2388a1320ef9b9ec17fbefbc4ab647ea98/pyee-14.0.0-py3-none-any.whl", hash = "sha256:3ac2d3229a9677f7de2c33d7f52fe25b638a46b19c413fea2edc8c6d0a644e4d", size = 15553, upload-time = "2026-08-13T04:26:09.916Z" }, +] + [[package]] name = "pygments" version = "2.21.0" diff --git a/docs/design.md b/docs/design.md index ebfbbd2..1bb2df8 100644 --- a/docs/design.md +++ b/docs/design.md @@ -160,7 +160,7 @@ CPU 不是问题,要省的是带宽,做法见第 14 节。 ### 7.3 频率和礼貌 - 平时每个来源每天抓 2 次;3–9 月和 12–1 月是旺季,每 4 小时一次。 -- 同一个网站同一时间只发一个请求,两次请求之间隔 5 秒;请求头写明是鼠钟,附上联系方式。 +- 同一个网站同一时间只发一个请求,两次请求之间隔 5 秒;请求头写明是鼠钟,附上联系方式。浏览器的一次页面访问算一个请求。 - 不读 robots.txt:抓的都是公开的通知列表,礼貌靠上面这几条保证;有的学院官网一请求 robots.txt 就断开连接(比如西电),照它的规矩反而一页都抓不了。 - 失败了等下一轮再抓,一轮里不反复重试。 diff --git a/docs/development.md b/docs/development.md index e8b195f..62c08e3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -59,13 +59,22 @@ crawler/ 抓取节点 - 接入来源时按通用抓取 → 配置 → 浏览器抓取 → 专用适配器的顺序试,前一种能抓就不用后一种([总体设计 §7.2](design.md#72-怎么抓))。 - 建站系统的模板放在代码里,和它的测试在一起;单个来源的配置存在网站的数据库里,在管理页面改,抓取节点领活时一起拿到。 -- 所有对来源网站的请求都经过 `crawl/fetch.py` 的 `Fetcher`,整个进程共用一个,[总体设计 §7.3](design.md#73-频率和礼貌)的规矩由它保证:UA 写明鼠钟;同一个网站一次一个请求,多个任务同时抓也一样;跳转的每一步单独发,跳到别的网站按那个网站的间隔排队。浏览器抓取打开页面前,也要先过这些规矩。 +- 所有对来源网站的请求都经过 `crawl/fetch.py` 的 `Fetcher`,整个进程共用一个,[总体设计 §7.3](design.md#73-频率和礼貌)的规矩由它保证:UA 写明鼠钟;同一个网站一次一个请求,多个任务同时抓也一样;跳转的每一步单独发,跳到别的网站按那个网站的间隔排队。浏览器一次页面访问取一次许可,持有到页面关闭;页面的子资源正常加载,跨网站跳转沿用起始网站的许可。 - 浏览器抓取启动时关掉 Chromium 的自动化标记(`--disable-blink-features=AutomationControlled`),UA 用浏览器自己的,把 `HeadlessChrome` 写成 `Chrome`,末尾加上鼠钟的标识;瑞数这类脚本挑战两处都检查([总体设计 §5.2](design.md#52-实测))。不冒充别的浏览器和系统。 - 一轮里同一个网站共用一个无痕会话,抓完就关:脚本挑战设的 Cookie 在会话里有效,后面的页面不用再过挑战;不同网站、不同轮次之间不共享 Cookie。同时开的浏览器页面有上限,多的排队;拒绝弹窗和下载,附件由程序另外下载;不加载图片、字体和音视频,省对方的流量,也省内存。 - 解析和模型分工:`parse` 用程序取出能确定的东西,模型只做判断([总体设计 §8.2](design.md#82-抽取))。解析不猜:判断不了的交给模型或人,不硬配。 - 判别模型和 LLM 的提示词放在调用它的代码旁边。输出都按固定的结构解析,解析不了的当失败处理。 - 每一轮抓取都交一份抓取记录,失败也交,写明原因;不在一轮里反复重试。各步抓不成时抛 `FetchError`(`Fetcher` 抛的)或 `StepError`(写明失败类别,比如 `http_status` 和「HTTP 403」),一轮的流程把它记进抓取记录;别的异常是 bug,照常抛出。 +在 `crawler/` 安装项目依赖、Chromium 无头精简版及系统依赖: + +```sh +uv sync +uv run playwright install --with-deps --only-shell chromium +``` + +CI 不装浏览器。自动测试用假的浏览器对象、固定响应和假时钟;真实浏览器和来源网站在本地手动验证,结果写进 PR。 + ### 2.1 列表配置 特殊列表通过 `ListConfig` 校验 `WorkSource.config`,管理员与 #46 的试抓生成使用同一格式。例如只能用 HTTP 的浙大计算机学院列表: