实现读正文和附件

crawler/pyproject.toml

@@ -5,9 +5,14 @@ description = "鼠钟的抓取节点：抓取、初筛和抽取"
 requires-python = ">=3.14"
 dependencies = [
     "httpx>=0.28",
+    "legacy-doc>=0.2.1",
     "lxml>=6.1.3",
+    "openpyxl>=3.1.5",
     "playwright>=1.64.1",
     "pydantic>=2.14.1",
+    "pypdf>=7.0.0",
+    "python-docx>=1.2.0",
+    "xlrd>=2.0.2",
 ]
 
 [dependency-groups]
@@ -17,6 +22,8 @@ dev = [
     "pytest>=9.1",
     "ruff>=0.17",
     "types-lxml>=2026.2.16",
+    "types-openpyxl>=3.1.5.20261017",
+    "types-xlrd>=2.0.0.20260518",
 ]
 
 [build-system]

crawler/src/crawler/crawl/notice.py

@@ -0,0 +1,85 @@
+import asyncio
+import logging
+import mimetypes
+import re
+
+from crawler.crawl.browser import PageSession
+from crawler.crawl.contract import AttachmentInfo, WorkSource
+from crawler.crawl.fetch import FetchError
+from crawler.crawl.rounds import Attachment, NoticePage, StepError
+from crawler.parse.files import file_text
+from crawler.parse.notice import AttachmentLink, parse_notice
+
+# What the website keeps (docs/api.md §6): 50 attachments to a snapshot, 2 MiB of text to one,
+# files up to 50 MB.
+MAX_ATTACHMENTS = 50
+MAX_TEXT = 2 * 1024 * 1024
+MAX_FILE = 50 * 1024 * 1024
+# Only the types Python itself knows, so every machine names them alike.
+_TYPES = mimetypes.MimeTypes()
+# An empty answer, or a web page in place of the file: an error, or a captcha the crawler doesn't
+# solve (design §7.2).
+_NOT_A_FILE = re.compile(rb"\s*(?:<|\Z)")
+logger = logging.getLogger(__name__)
+
+
+async def read_notice(pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None:
+    response = await pages.get(url)
+    if response.status in {404, 410}:
+        return None
+    if not 200 <= response.status < 300:
+        raise StepError("http_status", f"HTTP {response.status}")
+    try:
+        notice = parse_notice(response.body, response.url, response.content_type)
+    except ValueError as error:
+        raise StepError("error", str(error)) from error
+    if len(notice.attachments) > MAX_ATTACHMENTS:
+        logger.warning(
+            "Keeping %d of %d attachments",
+            MAX_ATTACHMENTS,
+            len(notice.attachments),
+            extra={"url": url},
+        )
+    attachments = [
+        await _attachment(pages, link, response.url)
+        for link in notice.attachments[:MAX_ATTACHMENTS]
+    ]
+    return NoticePage(notice.html, notice.text, notice.hyperlinks, attachments)
+
+
+# Listed whatever happens to the file, so the positions evidence refers to stay put.
+async def _attachment(pages: PageSession, link: AttachmentLink, page_url: str) -> Attachment:
+    content_type = (
+        _TYPES.guess_file_type(link.name)[0]
+        or _TYPES.guess_type(link.url)[0]
+        or "application/octet-stream"
+    )
+    data = await _download(pages, link.url, page_url)
+    text = ""
+    if data is not None:
+        try:
+            text = await asyncio.to_thread(file_text, data)
+        except ValueError as error:
+            logger.warning("Attachment unreadable: %s", error, extra={"url": link.url})
+    info = AttachmentInfo(
+        name=link.name, url=link.url, content_type=content_type, text=text[:MAX_TEXT]
+    )
+    return Attachment(info, data)
+
+
+async def _download(pages: PageSession, url: str, page_url: str) -> bytes | None:
+    try:
+        response = await pages.download(url, page_url=page_url)
+    except FetchError as error:
+        logger.warning("Attachment not downloaded: %s", error.failure, extra={"url": url})
+        return None
+    if response.status != 200 or _NOT_A_FILE.match(response.body) or len(response.body) > MAX_FILE:
+        logger.warning(
+            "Attachment not downloaded: HTTP %s, %s, %d bytes",
+            response.status,
+            response.content_type,
+            len(response.body),
+            extra={"url": url},
+        )
+        return None
+    return response.body

crawler/src/crawler/crawl/notice_test.py

@@ -0,0 +1,90 @@
+import io
+
+import docx
+import httpx
+import pytest
+
+from crawler.crawl.browser import PageSession
+from crawler.crawl.contract import WorkSource
+from crawler.crawl.fetch_test import FakeSites
+from crawler.crawl.notice import read_notice
+from crawler.crawl.rounds import NoticePage, StepError
+
+SOURCE = WorkSource(
+    id=7,
+    url="https://example.edu.cn/tzgg.htm",
+    section="通知公告",
+    browser=False,
+    config=None,
+    backfill=False,
+)
+PAGE = "https://example.edu.cn/info/1001/101.htm"
+
+
+def docx_file(*paragraphs: str) -> bytes:
+    document = docx.Document()
+    for paragraph in paragraphs:
+        document.add_paragraph(paragraph)
+    stream = io.BytesIO()
+    document.save(stream)
+    return stream.getvalue()
+
+
+def read(pages: dict[str, httpx.Response], url: str = PAGE) -> NoticePage | None:
+    return FakeSites(pages).run(
+        lambda fetcher: read_notice(PageSession(fetcher, None), SOURCE, url)
+    )
+
+
+@pytest.mark.parametrize("status", [404, 410])
+def test_a_page_that_is_gone_reads_as_none(status: int) -> None:
+    assert read({PAGE: httpx.Response(status)}) is None
+
+
+@pytest.mark.parametrize(
+    ("response", "failure", "detail"),
+    [
+        (httpx.Response(403), "http_status", "HTTP 403"),
+        (httpx.Response(200, html="<div>自编的正文</div>"), "error", "认不出正文"),
+    ],
+)
+def test_a_page_that_cant_be_read_fails_the_notice(
+    response: httpx.Response, failure: str, detail: str
+) -> None:
+    with pytest.raises(StepError) as caught:
+        read({PAGE: response})
+    assert (caught.value.failure, caught.value.detail) == (failure, detail)
+
+
+def test_attachments_keep_their_order_and_the_ones_not_downloaded_stay_listed() -> None:
+    body = (
+        '<div class="v_news_content"><p>自编的通知正文。</p>'
+        '<p><a href="files/form.docx">申请表.docx</a></p>'
+        '<p><a href="files/letter.doc">推荐信.doc</a></p>'
+        '<p><a href="files/list.pdf">名单.pdf</a></p></div>'
+    )
+    pages = {
+        # The notice moved; its links are relative to where it is now.
+        "https://example.edu.cn/old/101.htm": httpx.Response(301, headers={"location": PAGE}),
+        PAGE: httpx.Response(200, html=body),
+        "https://example.edu.cn/info/1001/files/form.docx": httpx.Response(
+            200, content=docx_file("请于9月20日前提交。")
+        ),
+        # Some sites answer with a captcha page in place of the file.
+        "https://example.edu.cn/info/1001/files/letter.doc": httpx.Response(
+            200, html="<p>请输入验证码下载附件</p>"
+        ),
+    }
+    page = read(pages, "https://example.edu.cn/old/101.htm")
+    assert page is not None
+    assert page.text == "自编的通知正文。\n申请表.docx\n推荐信.doc\n名单.pdf"
+    assert [(a.info.name, a.info.content_type, a.info.text) for a in page.attachments] == [
+        (
+            "申请表.docx",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "请于9月20日前提交。",
+        ),
+        ("推荐信.doc", "application/msword", ""),
+        ("名单.pdf", "application/pdf", ""),
+    ]
+    assert [a.data is not None for a in page.attachments] == [True, False, False]

crawler/src/crawler/crawl/rounds.py

@@ -27,6 +27,7 @@
 )
 from crawler.crawl.fetch import FetchError
 from crawler.crawl.website import Website
+from crawler.parse.indexing import Hyperlink
 
 logger = logging.getLogger(__name__)
 
@@ -43,14 +44,17 @@ def __init__(self, failure: Failure, detail: str | None = None) -> None:
 @dataclass(frozen=True)
 class Attachment:
     info: AttachmentInfo
-    data: bytes
+    # None when the file couldn't be downloaded; the snapshot lists it all the same.
+    data: bytes | None
 
 
 @dataclass(frozen=True)
 class NoticePage:
     html: str
     # The words of the notice; extraction quotes them, so they are what the reader sees.
     text: str
+    # Where `text` has links, their targets absolute against the page's final address.
+    hyperlinks: tuple[Hyperlink, ...]
     # In the order they appear; evidence refers to them by position.
     attachments: list[Attachment]
 
@@ -168,7 +172,8 @@ async def _fetch_notice(
         ),
     )
     for position in result.missing_files:
-        await website.attachment(result.snapshot_id, position, page.attachments[position].data)
+        if (data := page.attachments[position].data) is not None:
+            await website.attachment(result.snapshot_id, position, data)
     if result.extraction is None:
         return 0
     programs, usage = await steps.extract(page, result.extraction)

crawler/src/crawler/crawl/rounds_test.py

@@ -103,7 +103,7 @@ def page(text: str, files: int = 0) -> NoticePage:
         )
         for n in range(files)
     ]
-    return NoticePage(html=f"<p>{text}</p>", text=text, attachments=attachments)
+    return NoticePage(html=f"<p>{text}</p>", text=text, hyperlinks=(), attachments=attachments)
 
 
 async def extract(
@@ -127,10 +127,14 @@ def test_a_round_screens_the_new_fetches_what_needs_it_and_reports() -> None:
         extracted=frozenset({3}),
     )
 
+    behind_captcha = Attachment(
+        AttachmentInfo(name="表.doc", url="http://a/表.doc", content_type="doc", text=""), None
+    )
+    with_files = page("冬令营", files=1)
+    with_files.attachments.append(behind_captcha)
+
     async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage | None:
-        return {"http://a/1.htm": page("冬令营", files=1), "http://a/3.htm": page("夏令营")}.get(
-            url
-        )
+        return {"http://a/1.htm": with_files, "http://a/3.htm": page("夏令营")}.get(url)
 
     site.run(Steps(list_items, screen, read, extract))
 
@@ -154,6 +158,8 @@ async def read(_pages: PageSession, _source: WorkSource, url: str) -> NoticePage
         "content_type": "pdf",
         "text": "表",
     }
+    # The one behind a captcha is listed, and there is no file to send.
+    assert snapshot["attachments"][1]["url"] == "http://a/表.doc"
     assert site.called("PUT /snapshots/10/attachments/0") == [b"%PDF"]
     [run] = site.called("POST /runs")
     assert run == {

crawler/src/crawler/extraction/evaluate_test.py

@@ -14,7 +14,7 @@
 from crawler.extraction.llm import DEFAULT_MODELS, LLM, Result, Usage, cost_cny, make_call
 from crawler.extraction.metrics import Evaluation, pair_programs
 from crawler.extraction.numbered import SelectedExtraction, SelectedFields
-from crawler.extraction.pages import Page, article, read_page, save_page
+from crawler.extraction.pages import Page, read_page, save_page
 from crawler.extraction.schema import Extraction, LabelProgram, Notice
 from crawler.parse.indexing import NoticeIndex
 
@@ -228,19 +228,6 @@ async def never_respond(_: httpx.Request) -> httpx.Response:
     assert result.usage is None
 
 
-def test_article_excludes_navigation_scripts_and_resolves_body_link() -> None:
-    html = (
-        '<nav>Wrong deadline</nav><div class="v_news_content"><p>Apply now.</p>'
-        '<script>Untrusted code</script><a href="../apply">Portal</a></div><footer>Footer</footer>'
-    )
-    text, links = article(html, "https://example.edu.cn/news/1")
-    assert text.strip() == "Apply now.\nPortal"
-    assert len(links) == 1
-    assert links[0].href == "https://example.edu.cn/apply"
-    assert text[links[0].start : links[0].end] == "Portal"
-    assert "https://example.edu.cn/apply" not in text
-
-
 def test_reuse_only_uses_cached_body_and_response_without_network(tmp_path: Path) -> None:
     item = notice()
     dataset = tmp_path / "labels.json"