From dc9a9a96af5a33c8f643b2605229cc22d64d8499 Mon Sep 17 00:00:00 2001 From: JunieXD <1240840076@qq.com> Date: Mon, 9 Nov 2026 19:41:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E8=AF=BB=E6=AD=A3=E6=96=87?= =?UTF-8?q?=E5=92=8C=E9=99=84=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 初筛通过的通知要存成快照再交给抽取，快照里的文字就是核对时看的原文，所以正文要取得干净，附件要按页面上的顺序取全。 - 正文按建站系统的正文容器取（博达、WebPlus、TRS、GPower，加上北科研究生院和国科大的几种），认不出的记成「认不出正文」，不猜 - 正文里的链接记下文字的位置，地址按跳转后的网址解析成绝对地址，随 NoticePage 交给抽取 - 附件取正文里和附件区里的文件，以及正文里的图片；用 pypdf、python-docx、legacy-doc、openpyxl、xlrd 读出文字 - 下载不成的附件（比如要验证码）照样列出、文字留空，不上传文件 - 列表页和通知页共用一份解码网页的代码；抽取实验改用正式的读正文 - 比较结果写进总体设计 §5.2，选定的库写进 §5.1 --- crawler/pyproject.toml | 7 + crawler/src/crawler/crawl/notice.py | 85 ++++++ crawler/src/crawler/crawl/notice_test.py | 90 +++++++ crawler/src/crawler/crawl/rounds.py | 9 +- crawler/src/crawler/crawl/rounds_test.py | 14 +- .../src/crawler/extraction/evaluate_test.py | 15 +- crawler/src/crawler/extraction/pages.py | 97 +------ crawler/src/crawler/parse/files.py | 100 +++++++ crawler/src/crawler/parse/files_test.py | 88 +++++++ .../crawler/parse/fixtures/notice/README.md | 17 ++ .../crawler/parse/fixtures/notice/abody.html | 10 + .../parse/fixtures/notice/gpower-article.html | 12 + .../crawler/parse/fixtures/notice/gpower.html | 11 + .../crawler/parse/fixtures/notice/trs.html | 22 ++ .../parse/fixtures/notice/ucas-unit.html | 14 + .../crawler/parse/fixtures/notice/ucas.html | 13 + .../parse/fixtures/notice/ustb-graduate.html | 14 + .../crawler/parse/fixtures/notice/vsb.html | 25 ++ .../parse/fixtures/notice/webplus.html | 16 ++ crawler/src/crawler/parse/listing.py | 14 +- crawler/src/crawler/parse/listing_test.py | 2 +- crawler/src/crawler/parse/notice.py | 245 ++++++++++++++++++ crawler/src/crawler/parse/notice_test.py | 101 ++++++++ crawler/src/crawler/parse/webpage.py | 21 ++ crawler/typings/legacy_doc/__init__.pyi | 5 + crawler/uv.lock | 93 +++++++ docs/design.md | 18 +- docs/development.md | 1 + 28 files changed, 1033 insertions(+), 126 deletions(-) create mode 100644 crawler/src/crawler/crawl/notice.py create mode 100644 crawler/src/crawler/crawl/notice_test.py create mode 100644 crawler/src/crawler/parse/files.py create mode 100644 crawler/src/crawler/parse/files_test.py create mode 100644 crawler/src/crawler/parse/fixtures/notice/README.md create mode 100644 crawler/src/crawler/parse/fixtures/notice/abody.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/gpower-article.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/gpower.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/trs.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/ucas-unit.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/ucas.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/ustb-graduate.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/vsb.html create mode 100644 crawler/src/crawler/parse/fixtures/notice/webplus.html create mode 100644 crawler/src/crawler/parse/notice.py create mode 100644 crawler/src/crawler/parse/notice_test.py create mode 100644 crawler/src/crawler/parse/webpage.py create mode 100644 crawler/typings/legacy_doc/__init__.pyi diff --git a/crawler/pyproject.toml b/crawler/pyproject.toml index ac2d3b4..e7c5072 100644 --- a/crawler/pyproject.toml +++ b/crawler/pyproject.toml @@ -5,9 +5,14 @@ description = "鼠钟的抓取节点：抓取、初筛和抽取" requires-python = ">=3.14" dependencies = [ "httpx>=0.28", + "legacy-doc>=0.2.1", "lxml>=6.1.3", + "openpyxl>=3.1.5", "playwright>=1.64.1", "pydantic>=2.14.1", + "pypdf>=7.0.0", + "python-docx>=1.2.0", + "xlrd>=2.0.2", ] [dependency-groups] @@ -17,6 +22,8 @@ dev = [ "pytest>=9.1", "ruff>=0.17", "types-lxml>=2026.2.16", + "types-openpyxl>=3.1.5.20261017", + "types-xlrd>=2.0.0.20260518", ] [build-system] diff --git a/crawler/src/crawler/crawl/notice.py b/crawler/src/crawler/crawl/notice.py new file mode 100644 index 0000000..5bf6938 --- /dev/null +++ b/crawler/src/crawler/crawl/notice.py @@ -0,0 +1,85 @@ +import asyncio +import logging +import mimetypes +import re + +from crawler.crawl.browser import PageSession +from crawler.crawl.contract import AttachmentInfo, WorkSource +from crawler.crawl.fetch import FetchError +from crawler.crawl.rounds import Attachment, NoticePage, StepError +from crawler.parse.files import file_text +from crawler.parse.notice import AttachmentLink, parse_notice + +# What the website keeps (docs/api.md §6): 50 attachments to a snapshot, 2 MiB of text to one, +# files up to 50 MB. +MAX_ATTACHMENTS = 50 +MAX_TEXT = 2 * 1024 * 1024 +MAX_FILE = 50 * 1024 * 1024 +# Only the types Python itself knows, so every machine names them alike. +_TYPES = mimetypes.MimeTypes() +# An empty answer, or a web page in place of the file: an error, or a captcha the crawler doesn't +# solve (design §7.2). +_NOT_A_FILE = re.compile(rb"\s*(?:<|\Z)") +logger = logging.getLogger(__name__) + + +async def read_notice(pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None: + response = await pages.get(url) + if response.status in {404, 410}: + return None + if not 200 <= response.status < 300: + raise StepError("http_status", f"HTTP {response.status}") + try: + notice = parse_notice(response.body, response.url, response.content_type) + except ValueError as error: + raise StepError("error", str(error)) from error + if len(notice.attachments) > MAX_ATTACHMENTS: + logger.warning( + "Keeping %d of %d attachments", + MAX_ATTACHMENTS, + len(notice.attachments), + extra={"url": url}, + ) + attachments = [ + await _attachment(pages, link, response.url) + for link in notice.attachments[:MAX_ATTACHMENTS] + ] + return NoticePage(notice.html, notice.text, notice.hyperlinks, attachments) + + +# Listed whatever happens to the file, so the positions evidence refers to stay put. +async def _attachment(pages: PageSession, link: AttachmentLink, page_url: str) -> Attachment: + content_type = ( + _TYPES.guess_file_type(link.name)[0] + or _TYPES.guess_type(link.url)[0] + or "application/octet-stream" + ) + data = await _download(pages, link.url, page_url) + text = "" + if data is not None: + try: + text = await asyncio.to_thread(file_text, data) + except ValueError as error: + logger.warning("Attachment unreadable: %s", error, extra={"url": link.url}) + info = AttachmentInfo( + name=link.name, url=link.url, content_type=content_type, text=text[:MAX_TEXT] + ) + return Attachment(info, data) + + +async def _download(pages: PageSession, url: str, page_url: str) -> bytes | None: + try: + response = await pages.download(url, page_url=page_url) + except FetchError as error: + logger.warning("Attachment not downloaded: %s", error.failure, extra={"url": url}) + return None + if response.status != 200 or _NOT_A_FILE.match(response.body) or len(response.body) > MAX_FILE: + logger.warning( + "Attachment not downloaded: HTTP %s, %s, %d bytes", + response.status, + response.content_type, + len(response.body), + extra={"url": url}, + ) + return None + return response.body diff --git a/crawler/src/crawler/crawl/notice_test.py b/crawler/src/crawler/crawl/notice_test.py new file mode 100644 index 0000000..496831b --- /dev/null +++ b/crawler/src/crawler/crawl/notice_test.py @@ -0,0 +1,90 @@ +import io + +import docx +import httpx +import pytest + +from crawler.crawl.browser import PageSession +from crawler.crawl.contract import WorkSource +from crawler.crawl.fetch_test import FakeSites +from crawler.crawl.notice import read_notice +from crawler.crawl.rounds import NoticePage, StepError + +SOURCE = WorkSource( + id=7, + url="https://example.edu.cn/tzgg.htm", + section="通知公告", + browser=False, + config=None, + backfill=False, +) +PAGE = "https://example.edu.cn/info/1001/101.htm" + + +def docx_file(*paragraphs: str) -> bytes: + document = docx.Document() + for paragraph in paragraphs: + document.add_paragraph(paragraph) + stream = io.BytesIO() + document.save(stream) + return stream.getvalue() + + +def read(pages: dict[str, httpx.Response], url: str = PAGE) -> NoticePage | None: + return FakeSites(pages).run( + lambda fetcher: read_notice(PageSession(fetcher, None), SOURCE, url) + ) + + +@pytest.mark.parametrize("status", [404, 410]) +def test_a_page_that_is_gone_reads_as_none(status: int) -> None: + assert read({PAGE: httpx.Response(status)}) is None + + +@pytest.mark.parametrize( + ("response", "failure", "detail"), + [ + (httpx.Response(403), "http_status", "HTTP 403"), + (httpx.Response(200, html="
自编的正文
"), "error", "认不出正文"), + ], +) +def test_a_page_that_cant_be_read_fails_the_notice( + response: httpx.Response, failure: str, detail: str +) -> None: + with pytest.raises(StepError) as caught: + read({PAGE: response}) + assert (caught.value.failure, caught.value.detail) == (failure, detail) + + +def test_attachments_keep_their_order_and_the_ones_not_downloaded_stay_listed() -> None: + body = ( + '

自编的通知正文。

' + '

申请表.docx

' + '

推荐信.doc

' + '

名单.pdf

' + ) + pages = { + # The notice moved; its links are relative to where it is now. + "https://example.edu.cn/old/101.htm": httpx.Response(301, headers={"location": PAGE}), + PAGE: httpx.Response(200, html=body), + "https://example.edu.cn/info/1001/files/form.docx": httpx.Response( + 200, content=docx_file("请于9月20日前提交。") + ), + # Some sites answer with a captcha page in place of the file. + "https://example.edu.cn/info/1001/files/letter.doc": httpx.Response( + 200, html="

请输入验证码下载附件

" + ), + } + page = read(pages, "https://example.edu.cn/old/101.htm") + assert page is not None + assert page.text == "自编的通知正文。\n申请表.docx\n推荐信.doc\n名单.pdf" + assert [(a.info.name, a.info.content_type, a.info.text) for a in page.attachments] == [ + ( + "申请表.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "请于9月20日前提交。", + ), + ("推荐信.doc", "application/msword", ""), + ("名单.pdf", "application/pdf", ""), + ] + assert [a.data is not None for a in page.attachments] == [True, False, False] diff --git a/crawler/src/crawler/crawl/rounds.py b/crawler/src/crawler/crawl/rounds.py index 0e55045..cf9dc2a 100644 --- a/crawler/src/crawler/crawl/rounds.py +++ b/crawler/src/crawler/crawl/rounds.py @@ -27,6 +27,7 @@ ) from crawler.crawl.fetch import FetchError from crawler.crawl.website import Website +from crawler.parse.indexing import Hyperlink logger = logging.getLogger(__name__) @@ -43,7 +44,8 @@ def __init__(self, failure: Failure, detail: str | None = None) -> None: @dataclass(frozen=True) class Attachment: info: AttachmentInfo - data: bytes + # None when the file couldn't be downloaded; the snapshot lists it all the same. + data: bytes | None @dataclass(frozen=True) @@ -51,6 +53,8 @@ class NoticePage: html: str # The words of the notice; extraction quotes them, so they are what the reader sees. text: str + # Where `text` has links, their targets absolute against the page's final address. + hyperlinks: tuple[Hyperlink, ...] # In the order they appear; evidence refers to them by position. attachments: list[Attachment] @@ -168,7 +172,8 @@ async def _fetch_notice( ), ) for position in result.missing_files: - await website.attachment(result.snapshot_id, position, page.attachments[position].data) + if (data := page.attachments[position].data) is not None: + await website.attachment(result.snapshot_id, position, data) if result.extraction is None: return 0 programs, usage = await steps.extract(page, result.extraction) diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index f1a2f2a..e78635b 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -103,7 +103,7 @@ def page(text: str, files: int = 0) -> NoticePage: ) for n in range(files) ] - return NoticePage(html=f"

{text}

", text=text, attachments=attachments) + return NoticePage(html=f"

{text}

", text=text, hyperlinks=(), attachments=attachments) async def extract( @@ -127,10 +127,14 @@ def test_a_round_screens_the_new_fetches_what_needs_it_and_reports() -> None: extracted=frozenset({3}), ) + behind_captcha = Attachment( + AttachmentInfo(name="表.doc", url="http://a/表.doc", content_type="doc", text=""), None + ) + with_files = page("冬令营", files=1) + with_files.attachments.append(behind_captcha) + async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None: - return {"http://a/1.htm": page("冬令营", files=1), "http://a/3.htm": page("夏令营")}.get( - url - ) + return {"http://a/1.htm": with_files, "http://a/3.htm": page("夏令营")}.get(url) site.run(Steps(list_items, screen, read, extract)) @@ -154,6 +158,8 @@ async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage "content_type": "pdf", "text": "表", } + # The one behind a captcha is listed, and there is no file to send. + assert snapshot["attachments"][1]["url"] == "http://a/表.doc" assert site.called("PUT /snapshots/10/attachments/0") == [b"%PDF"] [run] = site.called("POST /runs") assert run == { diff --git a/crawler/src/crawler/extraction/evaluate_test.py b/crawler/src/crawler/extraction/evaluate_test.py index 3068dcd..4ede092 100644 --- a/crawler/src/crawler/extraction/evaluate_test.py +++ b/crawler/src/crawler/extraction/evaluate_test.py @@ -14,7 +14,7 @@ from crawler.extraction.llm import DEFAULT_MODELS, LLM, Result, Usage, cost_cny, make_call from crawler.extraction.metrics import Evaluation, pair_programs from crawler.extraction.numbered import SelectedExtraction, SelectedFields -from crawler.extraction.pages import Page, article, read_page, save_page +from crawler.extraction.pages import Page, read_page, save_page from crawler.extraction.schema import Extraction, LabelProgram, Notice from crawler.parse.indexing import NoticeIndex @@ -228,19 +228,6 @@ async def never_respond(_: httpx.Request) -> httpx.Response: assert result.usage is None -def test_article_excludes_navigation_scripts_and_resolves_body_link() -> None: - html = ( - '

Apply now.

' - 'Portal
' - ) - text, links = article(html, "https://example.edu.cn/news/1") - assert text.strip() == "Apply now.\nPortal" - assert len(links) == 1 - assert links[0].href == "https://example.edu.cn/apply" - assert text[links[0].start : links[0].end] == "Portal" - assert "https://example.edu.cn/apply" not in text - - def test_reuse_only_uses_cached_body_and_response_without_network(tmp_path: Path) -> None: item = notice() dataset = tmp_path / "labels.json" diff --git a/crawler/src/crawler/extraction/pages.py b/crawler/src/crawler/extraction/pages.py index 6613ec6..40b3305 100644 --- a/crawler/src/crawler/extraction/pages.py +++ b/crawler/src/crawler/extraction/pages.py @@ -1,33 +1,12 @@ import asyncio -import codecs -import re from datetime import UTC, datetime -from html.parser import HTMLParser from pathlib import Path -from urllib.parse import urljoin from pydantic import BaseModel, TypeAdapter from crawler.crawl.fetch import Fetcher from crawler.parse.indexing import Hyperlink - -CONTAINERS = { - "gp-content", - "abody", - "v_news_content", - "wp_articlecontent", - "TRS_Editor", - "TRS_UEDITOR", - "vsb_content", - "vsb_content_2", - "Zoom", - "zoom", - "article-content", - "item-page", - "article_content", -} -BLOCKS = {"p", "div", "br", "tr", "li", "h1", "h2", "h3", "section"} -VOID = {"br", "img", "input", "hr", "meta", "link", "wbr", "source", "embed", "area"} +from crawler.parse.notice import parse_notice class Page(BaseModel): @@ -42,65 +21,6 @@ class PageError(ValueError): pass -class ArticleParser(HTMLParser): - def __init__(self, url: str) -> None: - super().__init__(convert_charrefs=True) - self.url = url - self.stack: list[tuple[str, bool]] = [] - self.active = 0 - self.parts: list[str] = [] - self.suppressed = 0 - self.hyperlinks: list[Hyperlink] = [] - self.link_start: tuple[int, str] | None = None - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - values = dict(attrs) - names = set((values.get("class") or "").split()) | {values.get("id") or ""} - begins = bool(names & CONTAINERS) and not self.active - if tag in {"script", "style"}: - self.suppressed += 1 - if tag not in VOID: - self.stack.append((tag, begins)) - if begins: - self.active += 1 - if self.active and not self.suppressed: - if tag in BLOCKS: - self.parts.append("\n") - href = values.get("href") - if tag == "a" and href and not href.startswith(("javascript:", "#")): - self.link_start = (len("".join(self.parts)), urljoin(self.url, href)) - - def handle_endtag(self, tag: str) -> None: - if tag == "a" and self.link_start is not None: - start, href = self.link_start - end = len("".join(self.parts)) - if end > start: - self.hyperlinks.append(Hyperlink(start, end, href)) - self.link_start = None - if self.active and not self.suppressed and tag in BLOCKS: - self.parts.append("\n") - if tag in {"script", "style"} and self.suppressed: - self.suppressed -= 1 - for index in range(len(self.stack) - 1, -1, -1): - if self.stack[index][0] == tag: - self.active -= sum(begins for _, begins in self.stack[index:]) - del self.stack[index:] - break - - def handle_data(self, data: str) -> None: - if self.active and not self.suppressed: - self.parts.append(data) - - -def article(html: str, url: str) -> tuple[str, tuple[Hyperlink, ...]]: - parser = ArticleParser(url) - parser.feed(html) - text = "".join(parser.parts) - if not text.strip(): - raise PageError("article_container_not_found") - return text, tuple(parser.hyperlinks) - - def cached_page(url: str, path: Path) -> Page | None: if not path.exists(): return None @@ -123,18 +43,15 @@ async def read_page(url: str, path: Path, fetcher: Fetcher, *, reuse_only: bool) response = await fetcher.get(url) if response.status != 200: raise PageError(f"http_{response.status}") - declaration = ( - (response.content_type or "") + " " + response.body[:4096].decode("ascii", errors="ignore") - ) - match = re.search(r"charset\s*=\s*['\"]?([\w.-]+)", declaration, re.IGNORECASE) - encoding = codecs.lookup(match[1] if match else "utf-8-sig").name - html = response.body.decode(encoding) - text, hyperlinks = article(html, response.url) + try: + notice = parse_notice(response.body, response.url, response.content_type) + except ValueError as error: + raise PageError(str(error)) from error page = Page( url=url, final_url=response.url, - text=text, - hyperlinks=hyperlinks, + text=notice.text, + hyperlinks=notice.hyperlinks, fetched_at=datetime.now(UTC), ) await asyncio.to_thread(save_page, page, path) diff --git a/crawler/src/crawler/parse/files.py b/crawler/src/crawler/parse/files.py new file mode 100644 index 0000000..c55f508 --- /dev/null +++ b/crawler/src/crawler/parse/files.py @@ -0,0 +1,100 @@ +"""The words in the files notices attach (design §5.1): PDF, Word and Excel, the old binary formats +too. Images wait for text recognition (#47); archives and other files have none to read.""" + +import io +import unicodedata +import zipfile +from collections.abc import Iterable, Iterator + +import docx +import openpyxl +import xlrd +from docx.table import Table +from docx.text.paragraph import Paragraph +from legacy_doc import extract_text +from pypdf import PdfReader + +_PDF = b"%PDF" +_ZIP = b"PK\x03\x04" +_OLE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" +# Old Word files keep their text in a stream of this name; old Excel files have none. +_WORD = "WordDocument".encode("utf-16-le") +# Some PDFs map common characters to the Kangxi radicals that look like them, ⽣ for 生. +_RADICALS = {code: unicodedata.normalize("NFKC", chr(code)) for code in range(0x2F00, 0x2FD6)} + + +def file_text(data: bytes) -> str: + """Raises ValueError when the file is broken.""" + try: + if data.startswith(_PDF): + pages = PdfReader(io.BytesIO(data)).pages + return "\n".join(page.extract_text() for page in pages).translate(_RADICALS) + if data.startswith(_ZIP): + parts = set(zipfile.ZipFile(io.BytesIO(data)).namelist()) + if "word/document.xml" in parts: + return _docx(data) + if "xl/workbook.xml" in parts: + return _xlsx(data) + elif data.startswith(_OLE): + return extract_text(data).text if _WORD in data else _xls(data) + # The files come from the colleges' sites, and parsers fail on broken ones in all sorts of ways. + except Exception as error: + raise ValueError(type(error).__name__) from error + return "" + + +def _lines(lines: Iterable[str]) -> str: + # Blank lines, and the empty cells that end a row, carry nothing. + return "\n".join(line.rstrip("\t") for line in lines if line.strip()) + + +def _docx(data: bytes) -> str: + return _lines(_blocks(docx.Document(io.BytesIO(data)).iter_inner_content())) + + +def _blocks(blocks: Iterator[Paragraph | Table]) -> Iterator[str]: + for block in blocks: + if not isinstance(block, Table): + yield block.text + continue + for row in block.rows: + cells = row.cells + texts: list[str] = [] + index = 0 + # A merged cell repeats for each grid column it spans. + while index < len(cells): + # A cell's paragraphs, and any table inside it, stay on its row's line. + lines = _blocks(cells[index].iter_inner_content()) + texts.append(" ".join(line for line in lines if line.strip())) + index += cells[index].grid_span + yield "\t".join(texts) + + +def _xlsx(data: bytes) -> str: + book = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True) + lines: list[str] = [] + for sheet in book.worksheets: + lines.append(sheet.title) + lines += ( + "\t".join("" if value is None else str(value) for value in row) + for row in sheet.iter_rows(values_only=True) + ) + book.close() + return _lines(lines) + + +def _xls(data: bytes) -> str: + book = xlrd.open_workbook(file_contents=data) + + def value(cell: xlrd.sheet.Cell) -> str: + if cell.ctype == xlrd.XL_CELL_DATE: + return str(xlrd.xldate_as_datetime(float(cell.value), book.datemode)) + if cell.ctype == xlrd.XL_CELL_NUMBER and float(cell.value).is_integer(): + return str(int(cell.value)) + return str(cell.value) + + lines: list[str] = [] + for sheet in book.sheets(): + lines.append(sheet.name) + lines += ("\t".join(value(cell) for cell in sheet.row(row)) for row in range(sheet.nrows)) + return _lines(lines) diff --git a/crawler/src/crawler/parse/files_test.py b/crawler/src/crawler/parse/files_test.py new file mode 100644 index 0000000..c949177 --- /dev/null +++ b/crawler/src/crawler/parse/files_test.py @@ -0,0 +1,88 @@ +import io +import zipfile + +import docx +import openpyxl +import pytest + +from crawler.parse.files import file_text + + +def test_word_keeps_paragraphs_and_tables_in_order_with_merged_cells_once() -> None: + document = docx.Document() + document.add_paragraph("报名截止6月30日。") + table = document.add_table(rows=2, cols=3) + table.cell(0, 0).merge(table.cell(0, 1)).text = "专业与方向" + table.cell(0, 2).text = "名额" + for column, value in enumerate(["计算机", "人工智能", "10"]): + table.cell(1, column).text = value + table.cell(1, 2).add_table(rows=1, cols=1).cell(0, 0).text = "截止9月20日" + document.add_paragraph("") + document.add_paragraph("以上名额为预计。") + stream = io.BytesIO() + document.save(stream) + assert file_text(stream.getvalue()) == ( + "报名截止6月30日。\n专业与方向\t名额\n计算机\t人工智能\t10 截止9月20日\n以上名额为预计。" + ) + + +def test_excel_reads_each_sheet_by_rows() -> None: + book = openpyxl.Workbook() + sheet = book.active + assert sheet is not None + sheet.title = "专业目录" + for row in (["专业", "名额"], ["计算机", 10], [None, None], ["人工智能", 5]): + sheet.append(row) + book.create_sheet("联系方式").append(["电话", "010-00000000"]) + stream = io.BytesIO() + book.save(stream) + assert file_text(stream.getvalue()) == ( + "专业目录\n专业\t名额\n计算机\t10\n人工智能\t5\n联系方式\n电话\t010-00000000" + ) + + +def pdf_file(content: bytes, to_unicode: bytes) -> bytes: + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 100] /Contents 4 0 R" + b" /Resources << /Font << /F1 5 0 R >> >> >>", + b"<< /Length %d >>\nstream\n%s\nendstream" % (len(content), content), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /ToUnicode 6 0 R >>", + b"<< /Length %d >>\nstream\n%s\nendstream" % (len(to_unicode), to_unicode), + ] + data = b"%PDF-1.4\n" + offsets: list[int] = [] + for number, body in enumerate(objects, 1): + offsets.append(len(data)) + data += b"%d 0 obj\n%s\nendobj\n" % (number, body) + xref = len(data) + data += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + data += b"".join(b"%010d 00000 n \n" % offset for offset in offsets) + data += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n" % (len(objects) + 1, xref) + return data + b"%%EOF\n" + + +def test_pdf_characters_mapped_to_look_alike_radicals_read_as_the_common_ones() -> None: + # The font maps its glyph for 生 to the Kangxi radical ⽣ (U+2F63), as some PDFs do. + to_unicode = ( + b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap" + b" /CMapName /Example def 1 begincodespacerange <00> endcodespacerange" + b" 2 beginbfchar <41> <5B66> <42> <2F63> endbfchar" + b" endcmap CMapName currentdict /CMap defineresource pop end end" + ) + data = pdf_file(b"BT /F1 12 Tf 10 50 Td (AB) Tj ET", to_unicode) + assert file_text(data) == "学生" + + +def test_files_with_no_words_to_read_give_none() -> None: + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as files: + files.writestr("说明.txt", "自编的说明") + assert file_text(archive.getvalue()) == "" + assert file_text(b"\x89PNG\r\n\x1a\n") == "" + + +def test_a_broken_file_is_an_error() -> None: + with pytest.raises(ValueError, match="BadZipFile"): + file_text(b"PK\x03\x04" + b"\x00" * 30) diff --git a/crawler/src/crawler/parse/fixtures/notice/README.md b/crawler/src/crawler/parse/fixtures/notice/README.md new file mode 100644 index 0000000..d42827f --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/README.md @@ -0,0 +1,17 @@ +# 通知页结构样例 + +2026-11-09 通过 `Fetcher` 各读一页通知，只照抄正文和附件附近的结构。正文、标题、文件名、链接和图片地址全部换成自编的内容；没有通知原文、附件或学生信息。 + +| 样例 | 系统及结构依据 | 结构来源 | +| --- | --- | --- | +| `vsb.html` | Visual SiteBuilder（博达）：正文在 `v_news_content`；附件既有正文里带文件类型图标的链接，也有正文后 `ul` 里「附件【…】已下载 N 次」的列表，地址都是 `download.jsp?urltype=news.DownloadAttachUrl`；有的名称写成【附件2.xlsx】 | [北邮人工智能学院](https://ai.bupt.edu.cn/tzgg.htm)、[厦大信息学院](https://informatics.xmu.edu.cn/zsxx.htm)、[西电人工智能学院](https://sai.xidian.edu.cn/tzgg.htm) | +| `webplus.html` | WebPlus：正文在 `wp_articlecontent`；嵌入的 PDF 播放器只有 `pdfsrc` 和 `sudyfile-attr` 里的标题，正文为空；另一种是带 `sudy-attachment-icon` 的链接 | [东南大学电子学院](https://electronic.seu.edu.cn/11484/list.htm)、[东南大学研究生院](https://yzb.seu.edu.cn/6676/list.htm) | +| `trs.html` | TRS：正文在 `TRS_UEDITOR`，附件是 `insertfileTag` 里带 `download` 的链接；正文后的 `wrap-appendix` 附件区在 HTML 里是空的，附件的地址和名称写在页面脚本的 `appLinkStr`、`appDescStr` 里，由脚本填进去 | [自动化所](http://ia.cas.cn/yjsjy/zs/sszs/)、[信工所](http://iie.cas.cn/xsjy/zsxx/sszs/) | +| `gpower.html` | GPower：`gp-article` 里有标题、日期和 `gp-content` 正文 | [北科人工智能学院](https://ai.ustb.edu.cn/xwgg/tzgg/) | +| `gpower-article.html` | GPower 的另一种：正文直接在 `gp-article`，附件链接带 `download` | [北科自动化学院](https://saee.ustb.edu.cn/tzgg/index.htm) | +| `ustb-graduate.html` | 正文只标成 `wrapArticle` 里的 `article`，附件在正文后的 `Annex` | [北科研究生院](https://yzxc.ustb.edu.cn/xlytm/hdap/index.htm) | +| `ucas.html` | 国科大研招网的文章页：正文在 `b-abody`，外面的 `b-pagecontent` 还有发布单位和日期 | [国科大硕士招生](https://admission.ucas.ac.cn/showarticle/newslist/1beb211c-ea0d-4c97-b5ce-61efa7addfcd) | +| `ucas-unit.html` | 国科大研招网的招生单位页：没有 `b-abody`，正文是整个 `b-pagecontent` | [计算所硕士招生](https://www.ict.ac.cn/yjsjy/zsxx/sszs/)链到的招生目录 | +| `abody.html` | 国科大前沿交叉科学学院：正文在 `abody` | [国科大前沿交叉科学学院](https://sais.ucas.ac.cn/index.php/zh/xwgs/tgzs) | + +导航、页脚、侧栏的「常用下载」和二维码图片这些正文以外的链接，`vsb.html` 里留了几个，测试它们不被当成附件。下载、跳转、验证码页和失败的情况在 `crawl/notice_test.py` 里用自编的页面和现生成的附件测，不访问外网。 diff --git a/crawler/src/crawler/parse/fixtures/notice/abody.html b/crawler/src/crawler/parse/fixtures/notice/abody.html new file mode 100644 index 0000000..7eb814d --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/abody.html @@ -0,0 +1,10 @@ + + +自编通知 + +
+ + diff --git a/crawler/src/crawler/parse/fixtures/notice/gpower-article.html b/crawler/src/crawler/parse/fixtures/notice/gpower-article.html new file mode 100644 index 0000000..41171e3 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/gpower-article.html @@ -0,0 +1,12 @@ + + +自编通知 + +

示例学院拟录取名单公示

+
日期：2026-10-10
+

我院2027年接收推免生工作已经结束，拟录取名单见附件。

+

示例拟录取名单

+
+ + + diff --git a/crawler/src/crawler/parse/fixtures/notice/gpower.html b/crawler/src/crawler/parse/fixtures/notice/gpower.html new file mode 100644 index 0000000..12e5f97 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/gpower.html @@ -0,0 +1,11 @@ + + +自编通知 + +
+
示例学院2027年接收推免生工作办法
+
时间：2026-09-15来源：作者：示例学院
+

为做好2027年推免生接收工作，制定本办法。

一、组织管理

+
+ + diff --git a/crawler/src/crawler/parse/fixtures/notice/trs.html b/crawler/src/crawler/parse/fixtures/notice/trs.html new file mode 100644 index 0000000..415b175 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/trs.html @@ -0,0 +1,22 @@ + + +自编通知 + +
首页
+
+

示例研究所2027年接收推荐免试研究生，报名截止2026年9月20日。

+

附件1.示例报名表.doc

+
+
附件：
+ + +
联系我们
+ + diff --git a/crawler/src/crawler/parse/fixtures/notice/ucas-unit.html b/crawler/src/crawler/parse/fixtures/notice/ucas-unit.html new file mode 100644 index 0000000..8271961 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/ucas-unit.html @@ -0,0 +1,14 @@ + + +自编通知 + +
+
  • 示例研究所
+
+

示例研究所创建于1956年。

+
单位代码80000
+
+
+ + + diff --git a/crawler/src/crawler/parse/fixtures/notice/ucas.html b/crawler/src/crawler/parse/fixtures/notice/ucas.html new file mode 100644 index 0000000..0ca54af --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/ucas.html @@ -0,0 +1,13 @@ + + +自编通知 + +
+

示例大学2027年硕士研究生招生章程

+
+
  • 招生办公室
  • 2026年10月08日
+

示例大学2027年有若干个培养单位招收硕士研究生。

一、培养目标

+
+
+ + diff --git a/crawler/src/crawler/parse/fixtures/notice/ustb-graduate.html b/crawler/src/crawler/parse/fixtures/notice/ustb-graduate.html new file mode 100644 index 0000000..c1872fe --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/ustb-graduate.html @@ -0,0 +1,14 @@ + + +自编通知 + +
首页
+
+

欢迎优秀应届本科毕业生申请推荐免试攻读示例大学研究生。

一、申请条件

+
相关附件
+
+ + diff --git a/crawler/src/crawler/parse/fixtures/notice/vsb.html b/crawler/src/crawler/parse/fixtures/notice/vsb.html new file mode 100644 index 0000000..08718a6 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/vsb.html @@ -0,0 +1,25 @@ + + +自编通知 + +
首页常用下载
+
+
示例学院2027年接收推免生通知
+
+

为做好示例学院2027年推免生接收工作，现将有关事项通知如下。

+

一、报名时间：2026年9月1日至9月20日。

+

报名网址：报名系统，咨询邮箱 yz@example.edu.cn。

+
专业名额

计算机科学与技术

10
+

+

 附件1：示例申请表.doc

+ +
+
+

+
+
网站声明.pdf
+ + diff --git a/crawler/src/crawler/parse/fixtures/notice/webplus.html b/crawler/src/crawler/parse/fixtures/notice/webplus.html new file mode 100644 index 0000000..81a2845 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/notice/webplus.html @@ -0,0 +1,16 @@ + + +自编通知 + +
首页
+

示例学院推免生考核结果

+

发布时间：2026-09-10

+
+

示例学院2027年接收推免生考核结果如下。

+

+

示例拟录取名单.pdf

+
+
+
示例大学 版权所有
+ + diff --git a/crawler/src/crawler/parse/listing.py b/crawler/src/crawler/parse/listing.py index 7696e7b..cce5f98 100644 --- a/crawler/src/crawler/parse/listing.py +++ b/crawler/src/crawler/parse/listing.py @@ -1,14 +1,14 @@ -import codecs import re from dataclasses import dataclass from datetime import date from urllib.parse import urldefrag, urljoin, urlsplit -from lxml import etree, html +from lxml import etree from lxml.html import HtmlElement from pydantic import BaseModel, ConfigDict, field_validator from crawler.crawl.contract import ListedItem +from crawler.parse.webpage import parse_webpage class Selectors(BaseModel): @@ -147,15 +147,7 @@ def parse_listing( ) -> ListPage: if not body.strip(): return ListPage([], None) - declaration = (content_type or "") + " " + body[:4096].decode("ascii", errors="ignore") - match = re.search(r"charset\s*=\s*['\"]?([\w.-]+)", declaration, re.IGNORECASE) - try: - encoding = codecs.lookup(match[1] if match else "utf-8").name - root = html.document_fromstring( - body, parser=html.HTMLParser(encoding=encoding, no_network=True, remove_comments=True) - ) - except (LookupError, etree.ParserError) as error: - raise ValueError("列表页无法解析") from error + root = parse_webpage(body, content_type) try: if selectors is not None: items: dict[str, ListedItem] = {} diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py index 9c405f6..98c2eb8 100644 --- a/crawler/src/crawler/parse/listing_test.py +++ b/crawler/src/crawler/parse/listing_test.py @@ -456,7 +456,7 @@ def test_page_parse_failure_is_not_reported_as_a_configuration_error() -> None: ) with pytest.raises(StepError) as caught: sites.run(lambda f: list_items(PageSession(f, None), source())) - assert caught.value.detail == "列表页无法解析" + assert caught.value.detail == "网页无法解析" def test_repeated_links_can_be_the_list_rows() -> None: diff --git a/crawler/src/crawler/parse/notice.py b/crawler/src/crawler/parse/notice.py new file mode 100644 index 0000000..6dcb4fb --- /dev/null +++ b/crawler/src/crawler/parse/notice.py @@ -0,0 +1,245 @@ +"""A notice's page as its snapshot keeps it (design §7.4): the words of the body with the links in +them, the body's HTML, and the files the notice attaches.""" + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath +from urllib.parse import unquote, urldefrag, urljoin, urlsplit + +from lxml.html import HtmlElement, tostring + +from crawler.parse.indexing import Hyperlink +from crawler.parse.webpage import parse_webpage + +# The body as the site systems of the sources mark it, more specific first (design §7.2); +# fixtures/notice/README.md lists the systems and the sites they come from. +_BODIES = ( + "v_news_content", + "wp_articlecontent", + "TRS_UEDITOR", + "gp-content", + "gp-article", + "b-abody", + "b-pagecontent", + "abody", +) +# Where some systems list the attachments, apart from the body. +_ATTACHMENT_AREAS = ("Annex", "wrap-appendix", "cas-wrap-appendix") +# VSB's links to a notice's attachments, in the body or listed after it. +_VSB_ATTACHMENT = "DownloadAttachUrl" +_FILE = re.compile(r"\.(?:pdf|docx?|xlsx?|wps|pptx?|zip|rar|7z|jpe?g|png|gif|bmp)$", re.IGNORECASE) +# TRS writes a notice's attachments into its page script, which fills the attachment area in a +# browser: addresses and names, separated by |. They are read from the script, which never runs. +_TRS_HREFS = re.compile(r"appLinkStr\s*=\s*'([^']*)'") +_TRS_NAMES = re.compile(r"appDescStr\s*=\s*'([^']*)'") +# The editors' file-type icons beside attachment links. +_ICONS = ("/fileTypeImages/", "/_ueditor/") +_BLOCKS = { + "address", + "article", + "aside", + "blockquote", + "br", + "dd", + "div", + "dl", + "dt", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hr", + "li", + "main", + "nav", + "ol", + "p", + "pre", + "section", + "table", + "tbody", + "tfoot", + "thead", + "tr", + "ul", +} +_CELLS = {"td", "th"} +_HIDDEN = {"script", "style", "noscript", "template"} +# From weakest to strongest; of the gaps met between two words, the strongest is kept. +_GAPS = ("", " ", "\t", "\n") + + +@dataclass(frozen=True) +class AttachmentLink: + name: str + url: str + + +@dataclass(frozen=True) +class Notice: + html: str + text: str + hyperlinks: tuple[Hyperlink, ...] + # In the order they appear on the page. + attachments: list[AttachmentLink] + + +def parse_notice(body: bytes, url: str, content_type: str | None) -> Notice: + """`url` is where the page ended up after redirects; links resolve against it.""" + root = parse_webpage(body, content_type) + root.make_links_absolute(url, handle_failures="ignore") + content = _body(root) + if content is None: + raise ValueError("认不出正文") + text = _Text() + text.walk(content) + return Notice( + tostring(content, encoding="unicode", with_tail=False), + "".join(text.parts), + tuple(text.hyperlinks), + _attachments(root, content, url), + ) + + +def _body(root: HtmlElement) -> HtmlElement | None: + for name in _BODIES: + if found := root.find_class(name): + return found[0] + # USTB's graduate school marks its body only as the article in wrapArticle. + return next( + ( + element + for element in root.find_class("article") + if (parent := element.getparent()) is not None and "wrapArticle" in parent.classes + ), + None, + ) + + +def _attachments(root: HtmlElement, content: HtmlElement, url: str) -> list[AttachmentLink]: + in_body = set(content.iter()) + in_areas = { + element + for name in _ATTACHMENT_AREAS + for area in root.find_class(name) + for element in area.iter() + } + links: dict[str, str] = {} + for element in root.iter(): + if element.tag == "script": + script = element.text or "" + if (hrefs := _TRS_HREFS.search(script)) and (names := _TRS_NAMES.search(script)): + for href, name in zip(hrefs[1].split("|"), names[1].split("|"), strict=False): + if href: + target = urldefrag(urljoin(url, href)).url + links.setdefault(target, name.strip()[:500] or _name(element, target)) + continue + if element.tag == "img": + src = element.get("src") + if src and element in in_body and not any(icon in src for icon in _ICONS): + target = urldefrag(urljoin(url, src)).url + links.setdefault(target, _name(element, target)) + continue + # WebPlus embeds a PDF as a player with its address in pdfsrc. + href = element.get("pdfsrc") or (element.get("href") if element.tag == "a" else None) + if not href: + continue + target = urldefrag(urljoin(url, href)).url + if not (element in in_body or element in in_areas or _VSB_ATTACHMENT in target): + continue + name = _name(element, target) + if ( + element.get("pdfsrc") + or element.get("download") is not None + or _VSB_ATTACHMENT in target + or _FILE.search(unquote(urlsplit(target).path)) + or _FILE.search(name) + ): + links.setdefault(target, name) + # The website keeps addresses up to 2000 characters. + return [ + AttachmentLink(name, target) + for target, name in links.items() + if urlsplit(target).scheme in {"http", "https"} and len(target) <= 2000 + ] + + +def _name(element: HtmlElement, url: str) -> str: + embedded = re.search(r"'title':'([^']+)'", element.get("sudyfile-attr", "")) + for candidate in ( + element.get("download"), + element.get("title"), + element.text_content(), + embedded and embedded[1], + element.get("alt"), + unquote(PurePosixPath(urlsplit(url).path).name), + ): + name = " ".join((candidate or "").split()) + # VSB writes some names in brackets: 【附件1.docx】. + if bracketed := re.fullmatch(r"【(.+)】", name): + name = bracketed[1] + if name: + return name[:500] + return "附件" + + +class _Text: + """The words of an element as a reader sees them: blocks on lines of their own, table cells + apart by tabs, spaces collapsed; and where each link's words are.""" + + def __init__(self) -> None: + self.parts: list[str] = [] + self.length = 0 + self.gap = "" + # Where each run of words begins, so a link knows where its first word went. + self.starts: list[int] = [] + self.hyperlinks: list[Hyperlink] = [] + + def separate(self, gap: str) -> None: + self.gap = max(self.gap, gap, key=_GAPS.index) + + def add(self, words: str) -> None: + if words[:1].isspace(): + self.separate(" ") + if run := " ".join(words.split()): + if self.length and self.gap: + self.parts.append(self.gap) + self.length += len(self.gap) + self.gap = "" + self.starts.append(self.length) + self.parts.append(run) + self.length += len(run) + if words[-1:].isspace(): + self.separate(" ") + + def walk(self, element: HtmlElement, in_cell: bool = False) -> None: + if element.tag in _HIDDEN: + return + if element.tag in _CELLS: + gap = "\t" + elif element.tag in _BLOCKS: + # Paragraphs in a cell stay on its row's line. + gap = " " if in_cell else "\n" + else: + gap = "" + self.separate(gap) + first = len(self.starts) + self.add(element.text or "") + for child in element: + self.walk(child, in_cell or element.tag in _CELLS) + self.add(child.tail or "") + self.separate(gap) + href = element.get("href") if element.tag == "a" else None + if ( + href + and len(self.starts) > first + and urlsplit(href).scheme in {"http", "https", "mailto"} + ): + self.hyperlinks.append(Hyperlink(self.starts[first], self.length, href)) diff --git a/crawler/src/crawler/parse/notice_test.py b/crawler/src/crawler/parse/notice_test.py new file mode 100644 index 0000000..06cab45 --- /dev/null +++ b/crawler/src/crawler/parse/notice_test.py @@ -0,0 +1,101 @@ +from pathlib import Path + +import pytest + +from crawler.parse.notice import AttachmentLink, parse_notice + +FIXTURES = Path(__file__).parent / "fixtures" / "notice" +URL = "https://example.edu.cn/info/1001/101.htm" +VSB_FILE = "https://example.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1&wbfileid=" + + +@pytest.mark.parametrize( + ("name", "text", "attachments"), + [ + ( + "vsb", + "为做好示例学院2027年推免生接收工作\uff0c现将有关事项通知如下。\n" + "一、报名时间\uff1a2026年9月1日至9月20日。\n" + "报名网址\uff1a报名系统\uff0c咨询邮箱 yz@example.edu.cn。\n" + "专业\t名额\n" + "计算机科学与技术\t10\n" + "附件1\uff1a示例申请表.doc", + [ + ("example_1.png", "https://example.edu.cn/__local/1/AB/CD/example_1.png"), + ("附件1\uff1a示例申请表.doc", VSB_FILE + "101"), + ("附件2.xlsx", VSB_FILE + "102"), + ], + ), + ( + "webplus", + "示例学院2027年接收推免生考核结果如下。\n示例拟录取名单.pdf", + [ + ( + "示例考核结果.pdf", + "https://example.edu.cn/_upload/article/files/aa/bb/example/1.pdf", + ), + ( + "示例拟录取名单.pdf", + "https://example.edu.cn/_upload/article/files/aa/bb/example/2.pdf", + ), + ], + ), + ( + "trs", + "示例研究所2027年接收推荐免试研究生\uff0c报名截止2026年9月20日。\n附件1.示例报名表.doc", + [ + ( + "附件1.示例报名表.doc", + "https://example.edu.cn/info/1001/P020260901000000000001.doc", + ), + ("示例专业目录.pdf", "https://example.edu.cn/info/1001/P020260901000000000002.pdf"), + ], + ), + ("gpower", "为做好2027年推免生接收工作\uff0c制定本办法。\n一、组织管理", []), + ( + "gpower-article", + "我院2027年接收推免生工作已经结束\uff0c拟录取名单见附件。\n示例拟录取名单", + [("示例拟录取名单.pdf", "https://example.edu.cn/info/docs/2026-10/example.pdf")], + ), + ( + "ustb-graduate", + "欢迎优秀应届本科毕业生申请推荐免试攻读示例大学研究生。\n一、申请条件", + [ + ( + "附件1\uff1a示例专业目录.xlsx", + "https://example.edu.cn/docs/2026-09/example1.xlsx", + ), + ("附件2\uff1a示例鉴定表.doc", "https://example.edu.cn/docs/2026-09/example2.doc"), + ], + ), + ("ucas", "示例大学2027年有若干个培养单位招收硕士研究生。\n一、培养目标", []), + ("ucas-unit", "示例研究所创建于1956年。\n单位代码\t80000", []), + ("abody", "根据评审细则\uff0c现将推荐名单公示如下。\n序号\t姓名\n1\t张三\n示例学院", []), + ], +) +def test_site_system_templates_keep_the_body_and_its_attachments_in_order( + name: str, text: str, attachments: list[tuple[str, str]] +) -> None: + notice = parse_notice((FIXTURES / f"{name}.html").read_bytes(), URL, None) + assert notice.text == text + assert notice.attachments == [AttachmentLink(*attachment) for attachment in attachments] + + +def test_links_keep_where_their_words_are_and_resolve_against_the_final_address() -> None: + page = ( + '

请在' + "报名系统填报\uff0c或发邮件至" + 'yz@example.edu.cn。' + '打印

' + ) + notice = parse_notice(page.encode("gbk"), "https://example.edu.cn/info/1001/101.htm", None) + assert notice.text == "请在报名系统填报\uff0c或发邮件至yz@example.edu.cn。打印" + assert [(notice.text[link.start : link.end], link.href) for link in notice.hyperlinks] == [ + ("报名系统", "https://example.edu.cn/info/apply/"), + ("yz@example.edu.cn", "mailto:yz@example.edu.cn"), + ] + + +def test_a_page_no_template_knows_is_not_guessed_at() -> None: + with pytest.raises(ValueError, match="认不出正文"): + parse_notice("
自编的正文
".encode(), URL, None) diff --git a/crawler/src/crawler/parse/webpage.py b/crawler/src/crawler/parse/webpage.py new file mode 100644 index 0000000..94da961 --- /dev/null +++ b/crawler/src/crawler/parse/webpage.py @@ -0,0 +1,21 @@ +import codecs +import re + +from lxml import etree, html +from lxml.html import HtmlElement + + +def parse_webpage(body: bytes, content_type: str | None) -> HtmlElement: + """The page in the charset its response or its own head declares, UTF-8 when neither does.""" + declaration = (content_type or "") + " " + body[:4096].decode("ascii", errors="ignore") + match = re.search(r"charset\s*=\s*['\"]?([\w.-]+)", declaration, re.IGNORECASE) + try: + encoding = codecs.lookup(match[1] if match else "utf-8").name + return html.document_fromstring( + body, + parser=html.HTMLParser( + encoding=encoding, no_network=True, remove_comments=True, remove_pis=True + ), + ) + except (LookupError, etree.ParserError) as error: + raise ValueError("网页无法解析") from error diff --git a/crawler/typings/legacy_doc/__init__.pyi b/crawler/typings/legacy_doc/__init__.pyi new file mode 100644 index 0000000..084f0bd --- /dev/null +++ b/crawler/typings/legacy_doc/__init__.pyi @@ -0,0 +1,5 @@ +# legacy-doc ships no types; this covers the part parse/files.py uses. +class DocExtractionResult: + text: str + +def extract_text(document_bytes: bytes) -> DocExtractionResult: ... diff --git a/crawler/uv.lock b/crawler/uv.lock index 6ecf832..8476f81 100644 --- a/crawler/uv.lock +++ b/crawler/uv.lock @@ -111,9 +111,14 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "httpx" }, + { name = "legacy-doc" }, { name = "lxml" }, + { name = "openpyxl" }, { name = "playwright" }, { name = "pydantic" }, + { name = "pypdf" }, + { name = "python-docx" }, + { name = "xlrd" }, ] [package.dev-dependencies] @@ -123,14 +128,21 @@ dev = [ { name = "pytest" }, { name = "ruff" }, { name = "types-lxml" }, + { name = "types-openpyxl" }, + { name = "types-xlrd" }, ] [package.metadata] requires-dist = [ { name = "httpx", specifier = ">=0.28" }, + { name = "legacy-doc", specifier = ">=0.2.1" }, { name = "lxml", specifier = ">=6.1.3" }, + { name = "openpyxl", specifier = ">=3.1.5" }, { name = "playwright", specifier = ">=1.64.1" }, { name = "pydantic", specifier = ">=2.14.1" }, + { name = "pypdf", specifier = ">=7.0.0" }, + { name = "python-docx", specifier = ">=1.2.0" }, + { name = "xlrd", specifier = ">=2.0.2" }, ] [package.metadata.requires-dev] @@ -140,6 +152,8 @@ dev = [ { name = "pytest", specifier = ">=9.1" }, { name = "ruff", specifier = ">=0.17" }, { name = "types-lxml", specifier = ">=2026.2.16" }, + { name = "types-openpyxl", specifier = ">=3.1.5.20261017" }, + { name = "types-xlrd", specifier = ">=2.0.0.20260518" }, ] [[package]] @@ -171,6 +185,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/5b/f76dac052451e88fc0b622c3d996fa48637da9fa1e3c2b9ca2b84aead28a/datamodel_code_generator-0.86.0-py3-none-any.whl", hash = "sha256:1a3d6f0a14a33e334b4251d04e67b0b6ef05c96d4625fb99bd09d716e30ea084", size = 1605265, upload-time = "2026-10-18T15:43:27.808Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "genson" version = "1.4.0" @@ -329,6 +352,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "legacy-doc" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/99/ec/7cc1afd576a1508aafdd3d6e7cb126b993940772703d35243741fa49b220/legacy_doc-0.2.1.tar.gz", hash = "sha256:3573a034638af1ebbbe1a727baf18b2f69e6ad6439e4650be59aadc069a9bb42", size = 13011, upload-time = "2026-05-28T13:29:32.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/c0/c20afbf15f0f453be9202ce2b9e4b4f4456f1661a8413d437237ef428ef0/legacy_doc-0.2.1-py3-none-any.whl", hash = "sha256:51c428238efc48ef77b32fb7e9e0f35081eeb0a88e181dd3f793189dc63355e6", size = 11688, upload-time = "2026-05-28T13:29:31.821Z" }, +] + [[package]] name = "lxml" version = "6.1.3" @@ -508,6 +540,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/c8/12811c9b48fde162bb72b6f2e78fada9a247a09d7bb5be2050a5d099c77b/nodeenv-1.11.0-py2.py3-none-any.whl", hash = "sha256:edaa16e6c14d7cf395d75d4bbd5a26390f4dc06501a33b4e76282b02cc688a25", size = 34277, upload-time = "2026-09-26T11:29:19.933Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -671,6 +715,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] +[[package]] +name = "pypdf" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/19/3e8223d0618bd441eb2b3e105de83ce4187e76f5823d876f60a109ee9072/pypdf-7.0.0.tar.gz", hash = "sha256:061aa78dec270854c912dde80c6bc060486fa955399d643da27ea30afeb6571b", size = 7201622, upload-time = "2026-11-06T12:10:03.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/21/e83122a25e334c718a4da67004fa94acd7215fe9f8e63a9d7c7f979eb52b/pypdf-7.0.0-py3-none-any.whl", hash = "sha256:112a2c7cbabefd8dfe498f7a13977e575e762d58cdb0d832acb8886922602c6f", size = 413949, upload-time = "2026-11-06T12:10:01.205Z" }, +] + [[package]] name = "pyright" version = "1.1.414" @@ -700,6 +753,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "python-docx" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" }, +] + [[package]] name = "pytokens" version = "0.4.1" @@ -818,6 +884,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/5c/03ec9befbf4bb5309bfd576c6a5ac1c75633f78f6b64cf1f594e97cd3d23/types_lxml-2026.2.16-py3-none-any.whl", hash = "sha256:5dd81ffa54830e5f361988737c5f1d6a0ae48b2742790637ec560df790ea0401", size = 97040, upload-time = "2026-02-17T02:34:49.286Z" }, ] +[[package]] +name = "types-openpyxl" +version = "3.1.5.20261017" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/13/ce0b36e7e29ee31ec75d6718d48d28d2d48ad319ddf4665f81c8f3a8fc95/types_openpyxl-3.1.5.20261017.tar.gz", hash = "sha256:3deb3b6279b65ab379c126df0ee1fd27500c9628edb8d9667d73f637462917dc", size = 102026, upload-time = "2026-10-17T06:20:01.431Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/17/a091e90fde3e90eff9168354269626e210a4c3c688910d8c9176b8a08cb4/types_openpyxl-3.1.5.20261017-py3-none-any.whl", hash = "sha256:3315cfebad290cc7ade1fccaf3a46ee820b3a8e7df726ff4f18d30ab7f383d68", size = 165886, upload-time = "2026-10-17T06:19:59.944Z" }, +] + [[package]] name = "types-webencodings" version = "0.6.0.20261104" @@ -827,6 +902,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/3b/93a20758bf4b0a388a97672c7bd8212856bf744f6894fdaa8d2f877815cd/types_webencodings-0.6.0.20261104-py3-none-any.whl", hash = "sha256:a719903a309f35d43a99461e801df94af2def074dec2694423c7a2601ddcc45d", size = 8577, upload-time = "2026-11-04T08:13:57.726Z" }, ] +[[package]] +name = "types-xlrd" +version = "2.0.0.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/a8/2e0cff3a842f4f32dc5b1bec6e29fd13fc64124f7ea92368c2997e62738d/types_xlrd-2.0.0.20260518.tar.gz", hash = "sha256:238031699230d0bc3dbcf3b67fc440e6c3b56cccae82a512ad1b748e1a2eaf91", size = 14017, upload-time = "2026-05-18T06:04:27.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/f4/927ae5a8ba20f996ab26e83166207176ef19cf029d9a6b50150713725a42/types_xlrd-2.0.0.20260518-py3-none-any.whl", hash = "sha256:710c97c0a2dae288504a7c7037397b7a639563681b53890529c2342dd5194b42", size = 16891, upload-time = "2026-05-18T06:04:26.259Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -847,3 +931,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] + +[[package]] +name = "xlrd" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, +] diff --git a/docs/design.md b/docs/design.md index 8a592c9..a66ef73 100644 --- a/docs/design.md +++ b/docs/design.md @@ -81,7 +81,7 @@ | 数据库 | 服务器上 1Panel 的 PostgreSQL 18 | 和鼠洞共用实例，不加新服务 | | 抓取节点 | Python 3.14，uv 管依赖；Ruff 做 lint 和格式化，Pyright 的严格模式做类型检查，pytest 做测试 | 浏览器抓取、正文提取、PDF 和 Office 解析、中文的文字识别，Python 的库最全，代价是多一套工具链。3.15 刚发布，文字识别要用的 onnxruntime 还没有它的安装包。Pyright 和 VS Code 的 Pylance 是同一个引擎，编辑器里看到的错误和 CI 一致 | | 抓取 | httpx（异步）请求网页，Playwright 驱动浏览器（Chromium 的无头精简版），lxml 按 CSS 或 XPath 取出列表 | 各自领域的主流库 | -| 正文、附件和文字识别 | 候选：trafilatura 取正文，pypdf 或 pdfplumber 读 PDF，python-docx 和 openpyxl 读 Office，RapidOCR 识别文字 | 都关在 `parse` 模块里，换了不影响别处；在[读正文和附件](https://github.com/mousebaoyan/mouseclock/issues/40)和[文字识别](https://github.com/mousebaoyan/mouseclock/issues/47)的任务里用真实的附件比较后定。不用 AGPL 许可的 PyMuPDF | +| 正文、附件和文字识别 | 正文按建站系统的模板用 lxml 取；pypdf 读 PDF，python-docx 读 `.docx`，legacy-doc 读老的 `.doc`，openpyxl 读 `.xlsx`，xlrd 读 `.xls`；文字识别候选 RapidOCR | 都关在 `parse` 模块里，换了不影响别处。正文和附件用真实的通知和附件比较后定，结果见第 5.2 节；文字识别在[文字识别](https://github.com/mousebaoyan/mouseclock/issues/47)的任务里定。不用 AGPL 许可的 PyMuPDF | | 接口的类型 | 网站用 Zod 写接口的定义，生成 OpenAPI 文档；抓取节点从这份文档生成 Pydantic 模型 | 两边语言不同，接口的类型也只有一个来源。抓取节点用 Pydantic 校验网站的返回和模型的输出 | | 判别接口和 LLM | System One 的 SemIf；DeepSeek 官方接口的 V4.1 Flash（`deepseek-flash`），按编号抽取 | 判别见第 8.1 节，抽取比较见第 8.5 节 | @@ -107,6 +107,16 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 | 厦大信息学院的列表页 | 浏览器 0.7 秒；普通请求 0.22 秒、39KB，标题都在 HTML 里，用不着浏览器 | | 电科计算机学院 | 用了瑞数的反爬：先返回 202 和一段脚本，脚本设好 Cookie 后重新请求。Playwright 默认带着自动化标记（`navigator.webdriver` 为真），无头模式的 UA 里写着 `HeadlessChrome`，有一处就返回 400；关掉自动化标记、UA 里写成 `Chrome` 就能通过，UA 末尾照样写明鼠钟。首页 3.1 秒，内存峰值约 425MB；同一个会话里再开列表页，不用再过挑战 | +**正文和附件**（2026-11-09）：45 个来源各取 2 条最新的通知，加上中科院几个所的 7 条，共 97 页；从中下载 35 个附件，PDF、Word、Excel 都有。 + +| 测了什么 | 结果 | +| --- | --- | +| 取正文 | trafilatura 遇到正文只有一个嵌入 PDF 的页面（东南大学的 WebPlus）、正文是一张图片的页面（北邮），会把导航和页脚当成正文；找「文字最多的一块」，83 页里有 19 页认成了页脚或翻页。按建站系统的正文容器取（博达、WebPlus、TRS、GPower，加上北科研究生院和国科大的几种），97 页都认得出 | +| PDF | pypdf 和 pdfplumber 在 8 个 PDF 上取出的汉字一个不差；pypdf 把表格的一行留在同一行，pdfplumber 会把折行的单元格拆开；pypdf 快两三倍（89 页 0.97 秒对 2.89 秒，Apple M4）。东南大学的两个 PDF 把常用字映射成样子相同的康熙部首（「⽣」代替「生」），取出后换回常用字 | +| Word | python-docx 按段落和表格的顺序取 `.docx`，表格一行一行，横向合并的单元格只取一次。90 页通知里的 90 个附件链接（不算正文里的图片）有 26 个是老的 `.doc`，多是申请表：legacy-doc（纯 Python，MIT 许可）读的 7 份里，6 份和 LibreOffice 导出的汉字一致，剩下一份 LibreOffice 丢了整张表格，它取全了。它没带类型标注，`crawler/typings/` 里补了用到的部分。另一种做法是镜像里装 antiword（2005 年后没人维护）或 LibreOffice（几百 MB） | +| Excel | openpyxl 读 `.xlsx`，xlrd 读 `.xls`，样本里各 1 个都没问题 | +| 下载 | 7 次拿到的不是文件，是博达的「请输入验证码下载附件」页面（北科计通、厦大、电子科大信通、国科大杭州等）；先打开通知页、带着来源页下载也一样，是网站的设置。按第 7.2 节不破解：附件照样列出，文字留空，不存文件 | + **文字识别**：RapidOCR（PP-OCRv6）识别一张 20 行的中文页面约 1 秒，进程峰值约 690MB；抓取节点只有 4 核，会慢几倍。 抓取节点同时开 2 个浏览器页面、一次识别一张图，峰值约 1.4GB，3.6GB 内存够用（第 16 节）。 @@ -152,7 +162,7 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 | 浏览器抓取 | 要执行 JS 才能看到内容的 | 用浏览器打开再抓 | | 专用适配器 | 前三种都不行的极少数网站 | 单独写代码 | -- **按建站系统做模板**：不少高校官网用的是同几家建站系统，一个模板能管一批。 +- **按建站系统做模板**：不少高校官网用的是同几家建站系统，一个模板能管一批。通知的正文也按建站系统的正文容器取；哪个系统都对不上的，这条通知记成「认不出正文」，补上那个系统的模板，不猜哪一块是正文。 - **LLM 只在接入时用**：管理员在网站上填好列表页的地址，点「试抓」；抓取节点下一次领活时领到它，抓下来，让 LLM 认出列表、标题、日期和翻页在哪，把抓到的条目和生成的配置交回网站。管理员看过没问题再保存；之后每天按配置抓，不再问 LLM。 - **试抓的流程**：管理员在来源管理页的表单里填好网址（以及要不要浏览器、已有的配置），点「试抓」，网站记一条试抓；页面显示「等抓取节点」，每隔几秒刷新，直到抓取节点交回结果。结果列出抓到的条目（标题、链接、日期）和生成的配置，失败的写原因。管理员看过点「采用」，网址、要不要浏览器和生成的配置填回表单，再保存来源：采用就是保存来源，配置改前改后记进操作日志；发起试抓也记一条。 - **不硬闯**：要登录才能看的、有验证码的不抓，标成「暂时抓不了」并写明原因。只要求用浏览器打开的脚本挑战（比如瑞数的反爬）不算：页面是公开的，人用浏览器打开就能看，浏览器抓取按[开发约定 §2](development.md#2-抓取和判别)的设置通过，UA 照样写明鼠钟。 @@ -170,7 +180,7 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 - **新通知**：列表里出现没见过的链接，直接比网址。 - **正文变化**：还没截止的项目，每天重抓一次它的通知，在它的来源那一轮里一起抓；正文的文字和上一份快照不同，就存一份新快照，再交给抽取。 - **删除**：重抓时通知打不开（404、410），标成「原文已删除」，快照照样能看。从列表里消失不算：列表会翻页，旧通知本来就会被挤下去。 -- **附件**：PDF、Word、Excel 和图片都下载下来，解析出文字，图片做文字识别；附件和正文一起存进快照。 +- **附件**：正文里和正文旁附件区里的文件（PDF、Word、Excel、压缩包），以及正文里的图片，按出现的顺序下载下来、解析出文字；图片做文字识别，压缩包不取文字。附件和正文一起存进快照。下载不成的（比如要验证码才能下载），照样列出名称和网址，文字留空。 ### 7.5 抓取看板 @@ -593,7 +603,7 @@ L3 以上可以不提申请，直接修改，同样要写理由、附依据。 | 事项 | 现状 | | --- | --- | | 名称 | 「鼠钟」暂定，帖子里在征名 | -| 正文、附件和文字识别用的库 | 用真实的附件比较后定（第 5.1 节） | +| 文字识别用的库 | 用真实的图片和扫描件比较后定（第 5.1 节） | | 域名 | 待定，用 `mousebaoyan.cn` 的子域名 | | 群汇总由哪个插件发 | 扩展鼠洞的 bot 插件，还是新写一个 | | 第一批来源 | 先收对四非友好的学校，名单在帖子里征集 | diff --git a/docs/development.md b/docs/development.md index 278e884..bf9d70d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -68,6 +68,7 @@ crawler/ 抓取节点 - 接入来源时按通用抓取 → 配置 → 浏览器抓取 → 专用适配器的顺序试，前一种能抓就不用后一种（[总体设计 §7.2](design.md#72-怎么抓)）。 - 建站系统的模板放在代码里，和它的测试在一起；单个来源的配置存在网站的数据库里，在管理页面改，抓取节点领活时一起拿到。 +- 读通知时，正文按建站系统的正文容器取（`parse/notice.py`），认不出的抛 `StepError`，写明「认不出正文」，补模板，不退回去猜。附件下载或解析不成的，照样列出、文字留空，记一条日志，不让整条通知失败。 - 所有对来源网站的请求都经过 `crawl/fetch.py` 的 `Fetcher`，整个进程共用一个，[总体设计 §7.3](design.md#73-频率和礼貌)的规矩由它保证：UA 写明鼠钟；同一个网站一次一个请求，多个任务同时抓也一样；跳转的每一步单独发，跳到别的网站按那个网站的间隔排队。浏览器一次页面访问取一次许可，持有到页面关闭；页面的子资源正常加载，跨网站跳转沿用起始网站的许可。 - 浏览器抓取启动时关掉 Chromium 的自动化标记（`--disable-blink-features=AutomationControlled`），UA 用浏览器自己的，把 `HeadlessChrome` 写成 `Chrome`，末尾加上鼠钟的标识；瑞数这类脚本挑战两处都检查（[总体设计 §5.2](design.md#52-实测)）。不冒充别的浏览器和系统。 - 一轮里同一个网站共用一个无痕会话，抓完就关：脚本挑战设的 Cookie 在会话里有效，后面的页面不用再过挑战；不同网站、不同轮次之间不共享 Cookie。同时开的浏览器页面有上限，多的排队；拒绝弹窗和下载，附件由程序另外下载；不加载图片、字体和音视频，省对方的流量，也省内存。