实现通知列表解析、配置和历史翻页

crawler/pyproject.toml

@@ -5,6 +5,7 @@ description = "鼠钟的抓取节点：抓取、初筛和抽取"
 requires-python = ">=3.14"
 dependencies = [
     "httpx>=0.28",
+    "lxml>=6.1.3",
     "pydantic>=2.14.1",
 ]
 
@@ -14,6 +15,7 @@ dev = [
     "pyright>=1.1.400",
     "pytest>=9.1",
     "ruff>=0.17",
+    "types-lxml>=2026.2.16",
 ]
 
 [build-system]

crawler/src/crawler/crawl/listing.py

@@ -0,0 +1,65 @@
+import time
+from calendar import monthrange
+from collections.abc import Callable
+from datetime import datetime, timedelta, timezone
+from urllib.parse import urlsplit, urlunsplit
+
+from pydantic import ValidationError
+
+from crawler.crawl.contract import ListedItem, WorkSource
+from crawler.crawl.fetch import Fetcher
+from crawler.crawl.rounds import StepError
+from crawler.parse.listing import ListConfig, parse_listing
+
+
+async def list_items(
+    fetcher: Fetcher, source: WorkSource, clock: Callable[[], float] = time.time
+) -> list[ListedItem]:
+    try:
+        config = ListConfig.model_validate(source.config or {})
+    except ValidationError as error:
+        location = ".".join(str(value) for value in error.errors(include_input=False)[0]["loc"])
+        raise StepError("error", f"配置不对: {location}") from error
+
+    def request_url(url: str) -> str:
+        parts = urlsplit(url)
+        if config.force_http and parts.netloc == urlsplit(source.url).netloc:
+            return urlunsplit(parts._replace(scheme="http"))
+        return url
+
+    now = datetime.fromtimestamp(clock(), timezone(timedelta(hours=8))).date()
+    year = now.year - 2
+    cutoff = now.replace(year=year, day=min(now.day, monthrange(year, now.month)[1]))
+    current = request_url(source.url)
+    visited: set[str] = set()
+    items: dict[str, ListedItem] = {}
+    while True:
+        if current in visited:
+            break
+        visited.add(current)
+        response = await fetcher.get(current)
+        visited.add(response.url)
+        if not 200 <= response.status < 300:
+            raise StepError("http_status", f"HTTP {response.status}")
+        try:
+            page = parse_listing(
+                response.body, response.url, config.selectors, response.content_type
+            )
+        except ValueError as error:
+            raise StepError("error", str(error)) from error
+        if not page.items:
+            break
+        fresh = [item for item in page.items if item.url not in items]
+        if (
+            source.backfill
+            and fresh
+            and all(item.published_on is not None and item.published_on < cutoff for item in fresh)
+        ):
+            break
+        for item in page.items:
+            if not source.backfill or item.published_on is None or item.published_on >= cutoff:
+                items.setdefault(item.url, item)
+        if not source.backfill or page.next_url is None:
+            break
+        current = request_url(page.next_url)
+    return list(items.values())

crawler/src/crawler/parse/fixtures/listing/README.md

@@ -0,0 +1,24 @@
+# 列表结构样例
+
+2026-11-07 通过原版 `Fetcher` 各取一页，只截取列表及翻页结构。标题、摘要、日期、通知链接和图片地址全部替换为自编内容；没有通知正文、快照、附件或学生信息。样例中日期均为 2026-09-10、2026-09-09。
+
+| 样例 | 系统及结构依据 | 结构来源 |
+| --- | --- | --- |
+| `vsb.html` | Visual SiteBuilder，原页有 `Announced by Visual SiteBuilder 9`；日期为日、年月分栏，标题与摘要分开 | [厦大信息学院招生信息](https://informatics.xmu.edu.cn/zsxx.htm) |
+| `vsb-calendar.html` | 同一建站系统的另一种日期布局，链接有完整 `title` 属性 | [西电电子工程学院研究生列表](https://see.xidian.edu.cn/yjs/158.htm) |
+| `webplus.html` | WebPlus，原页的 `webplus-list` 及 Sudy 资源；`news_list`、`news_title`、`news_meta` | [东南大学网安硕士生招生](https://cyber.seu.edu.cn/18210/list.htm) |
+| `trs.html` | TRS 类静态发布，`YYYYMM/tYYYYMMDD_编号.html`，日期在独立 `pull-right` 中；包含一条跳往研招系统的置顶链接 | [计算所硕士招生](https://www.ict.ac.cn/yjsjy/zsxx/sszs/) |
+| `gpower.html` | GPower 页面，原页的 `gpowersoft.js` 和 `/cms/web/`；`infoBox` 内标题、摘要、日期分开 | [北科大人工智能学院通知](https://ai.ustb.edu.cn/xwgg/tzgg/) |
+| `drupal.html` | 原页明确声明 Drupal 10，Views 容器、`list-page`、`time/@datetime`；只有页码的翻页器 | [中大计算机学院硕士招生](https://cse.sysu.edu.cn/admission/master) |
+| `configured.html` | 独立招生发布系统，作为需要配置的页面测试，不计入前述五种通用系统 | [国科大硕士招生](https://admission.ucas.ac.cn/showarticle/newslist/1beb211c-ea0d-4c97-b5ce-61efa7addfcd) |
+| `zju-http.html` | WebPlus 的另一种布局，使用开发约定中的 XPath 配置和显式 HTTP，列出 `jzlb`、`btt3`、`fbsj4` | [浙大计算机招生信息](http://www.cs.zju.edu.cn/csen/27006/list.htm) |
+| `opaque-links.html` | 通知链接为不带 `info/` 的字母数字文件名 | [北科大研究生院](https://yzxc.ustb.edu.cn/xlytm/hdap/index.htm) |
+| `month-day-year.html` | 月日与年份分栏，标题和摘要在同一链接内 | [北科大计通学院](https://scce.ustb.edu.cn/xydt/tzgg.htm) |
+| `year-month-day.html` | 年份与月日分栏 | [电子科大信通学院](https://www.sice.uestc.edu.cn/index/tzgg/yjsk.htm) |
+| `table-rows.html` | 每条通知使用嵌套表格的行排版 | [东南大学研究生院](https://yzb.seu.edu.cn/6676/list.htm) |
+| `date-in-link.html` | 日期在标题链接内部 | [西电人工智能学院](https://sai.xidian.edu.cn/tzgg.htm) |
+| `plain-span.html` | 日期在无 class 的 `span` 内 | [西电研究生院](https://gr.xidian.edu.cn/yjsy/yjszs.htm) |
+| `empty-link.html` | 通知链接前有一个空链接 | [西电网信学院](https://ce.xidian.edu.cn/sy/tzgg.htm) |
+| `right-span.html` | 日期在 `right` 元素内 | [沈自](https://sia.cas.cn/zpjy/yjsjy/zs/zsgg/) |
+
+Drupal 样例用于补充不同系统的解析覆盖，不改动 `data/sources.csv`。分页边界、未知日期、重复置顶、重定向、HTTP、非法配置和失败场景使用测试里自编的页面及 `MockTransport`，不访问外网。

crawler/src/crawler/parse/fixtures/listing/configured.html

@@ -0,0 +1,14 @@
+<html>
+<head><meta charset="utf-8"></head>
+<body>
+<nav><a href="/contact.htm">自编导航</a></nav><main><ul class="b-list">&#13;
+                                <li>
+<img src="/example.png"><span class="m-date">2026-09-10</span><a href="/showarticle/Article/example/101">示例招生通知甲</a>
+</li>
+<li>
+<img src="/example.png"><span class="m-date">2026-09-09</span><a href="/showarticle/Article/example/102">示例招生通知乙</a>
+</li>
+</ul>&#13;
+                        </main>
+</body>
+</html>

crawler/src/crawler/parse/fixtures/listing/date-in-link.html

@@ -0,0 +1,17 @@
+<meta charset="utf-8"><nav><a href="/nav">自编导航</a></nav><ul><li>
+
+                    <a href="/article/101.htm">自编招生通知1<span>2026-09-10</span></a>
+
+
+
+            </li>
+
+        <li>
+
+                    <a href="/article/102.htm">自编招生通知2<span>2026-09-9</span></a>
+
+
+
+            </li>
+
+        </ul>

crawler/src/crawler/parse/fixtures/listing/drupal.html

@@ -0,0 +1,58 @@
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="Generator" content="Drupal 10 (https://www.drupal.org)">
+</head>
+<body>
+<nav><a href="/contact.htm">自编导航</a></nav><main><div class="views-element-container"><div class="termlists">
+<ul>
+
+          <li class="list-page wenzi">
+<div class="posttime">
+<span class="day"><time datetime="2026-09-10T10:00:00+08:00">10</time></span><span class="year"><time datetime="2026-09-10T10:00:00+08:00">2026-09</time></span>
+</div>
+<div class="title"><a href="/article/101" target="_blank"><span class="text">示例招生通知甲</span><span class="summary">这是自编摘要,不能并入标题。</span></a></div>
+</li>
+<li class="list-page wenzi">
+<div class="posttime">
+<span class="day"><time datetime="2026-09-09T10:00:00+08:00">09</time></span><span class="year"><time datetime="2026-09-09T10:00:00+08:00">2026-09</time></span>
+</div>
+<div class="title"><a href="/article/102" target="_blank"><span class="text">示例招生通知乙</span><span class="summary">这是自编摘要,不能并入标题。</span></a></div>
+</li>
+</ul>
+
+</div></div>
+<div class="pager outside-tb t-c" role="navigation" aria-labelledby="pagination-heading">
+    <h4 id="pagination-heading" class="visually-hidden">分页</h4>
+    <ul class="menu pager-items pager-row js-pager-items">
+                                                  <li class="pager-item-number pager-item is-active">
+                                          <a href="?page=0" title="当前页" aria-current="page">
+            <span class="visually-hidden">
+              当前页
+            </span>1</a>
+        </li>
+              <li class="pager-item-number pager-item">
+                                          <a href="?page=1" title="转到页面2">
+            <span class="visually-hidden">
+              页面
+            </span>2</a>
+        </li>
+                                            <li class="pager-item pager-item-last">
+          <a href="?page=1" title="转到最后一页">
+            <span class="visually-hidden">末页</span>
+            <span aria-hidden="true">末页</span>
+            <i class="icon"></i>
+          </a>
+        </li>
+          </ul>
+  </div>
+
+
+
+
+
+
+
+</main>
+</body>
+</html>

crawler/src/crawler/parse/fixtures/listing/empty-link.html

@@ -0,0 +1,29 @@
+<meta charset="utf-8"><nav><a href="/nav">自编导航</a></nav><ul><li><a href="tzgg.htm">
+
+        </a><a href="/article/101.htm" title="自编招生通知1">
+
+            <p>自编招生通知1</p>
+
+            <i>2026-09-10</i>
+
+            <span>自编摘要，不能并入标题。</span>
+
+        </a>
+
+    </li>
+
+    <li><a href="tzgg.htm">
+
+        </a><a href="/article/102.htm" title="自编招生通知2">
+
+            <p>自编招生通知2</p>
+
+            <i>2026-09-9</i>
+
+            <span>自编摘要，不能并入标题。</span>
+
+        </a>
+
+    </li>
+
+    </ul>

crawler/src/crawler/parse/fixtures/listing/gpower.html

@@ -0,0 +1,23 @@
+<html>
+<head><meta charset="utf-8"></head>
+<body>
+<nav><a href="/contact.htm">自编导航</a></nav><main><ul>
+                                        <li><a href="11111111111111111111111111111111.htm" class="flex"><div class="imgBox"><div class="gp-img-responsive"><img src="/example.png" alt="示例图片"></div></div>
+<div class="infoBox">
+<div class="title gp-ellipsis gp-f24">示例招生通知甲</div>
+<div class="summary gp-ellipsis-2 gp-f16">这是自编摘要,不能并入标题。</div>
+<div class="date">2026-09-10<i></i><i></i>
+</div>
+</div></a></li>
+<li><a href="22222222222222222222222222222222.htm" class="flex"><div class="imgBox"><div class="gp-img-responsive"><img src="/example.png" alt="示例图片"></div></div>
+<div class="infoBox">
+<div class="title gp-ellipsis gp-f24">示例招生通知乙</div>
+<div class="summary gp-ellipsis-2 gp-f16">这是自编摘要,不能并入标题。</div>
+<div class="date">2026-09-09<i></i><i></i>
+</div>
+</div></a></li>
+</ul>
+                <a href="page2.htm" class="gp-page-next">下一页</a>
+                                            </main>
+</body>
+</html>

crawler/src/crawler/parse/fixtures/listing/month-day-year.html

@@ -0,0 +1,53 @@
+<meta charset="utf-8"><nav><a href="/nav">自编导航</a></nav><ul><li>
+
+    <a href="/article/101.htm" class="flex-middle">
+
+        <div class="date1">
+
+             <p class="p1">09.10</p>
+
+                <p class="p2">2026</p>
+
+        </div>
+
+        <div class="line"></div>
+
+        <div class="text">
+
+            <p class="title tt1">自编招生通知1</p>
+
+  <p class="info tt1">自编摘要，截止2030年1月1日。</p>
+
+        </div>
+
+    </a>
+
+</li>
+
+<li>
+
+    <a href="/article/102.htm" class="flex-middle">
+
+        <div class="date1">
+
+             <p class="p1">09.9</p>
+
+                <p class="p2">2026</p>
+
+        </div>
+
+        <div class="line"></div>
+
+        <div class="text">
+
+            <p class="title tt1">自编招生通知2</p>
+
+  <p class="info tt1">自编摘要，截止2030年1月1日。</p>
+
+        </div>
+
+    </a>
+
+</li>
+
+</ul>

crawler/src/crawler/parse/fixtures/listing/opaque-links.html

@@ -0,0 +1,9 @@
+<meta charset="utf-8"><nav><a href="/nav">自编导航</a></nav><ul><li><div class="box">
+							<div class="time fs14">2026-09-10</div>
+							<div class="title fs20"><a href="/article/101.htm">自编招生通知1</a></div>
+						</div></li>
+						                                                 <li><div class="box">
+							<div class="time fs14">2026-09-9</div>
+							<div class="title fs20"><a href="/article/102.htm">自编招生通知2</a></div>
+						</div></li>
+						                                                 </ul>

crawler/src/crawler/parse/fixtures/listing/plain-span.html

@@ -0,0 +1,13 @@
+<meta charset="utf-8"><nav><a href="/nav">自编导航</a></nav><ul><li>
+
+                    <a href="/article/101.htm">自编招生通知1</a><span>2026-09-10</span></li>
+
+
+
+        <li>
+
+                    <a href="/article/102.htm">自编招生通知2</a><span>2026-09-9</span></li>
+
+
+
+        </ul>

crawler/src/crawler/parse/fixtures/listing/right-span.html

@@ -0,0 +1,5 @@
+<meta charset="utf-8"><nav><a href="/nav">自编导航</a></nav><ul><li><a href="/article/101.htm">自编招生通知1</a><span class="right">2026-09-10</span></li>
+
+				    <li><a href="/article/102.htm">自编招生通知2</a><span class="right">2026-09-9</span></li>
+
+				    </ul>

crawler/src/crawler/parse/fixtures/listing/table-rows.html

@@ -0,0 +1,41 @@
+<meta charset="utf-8"><nav><a href="/nav">自编导航</a></nav><tr><td>
+
+<table>
+
+<tr>
+
+<td><a href="/article/101.htm" title="自编招生通知1">自编招生通知1</a></td>
+
+
+
+<td>2026-09-10<div></div></td>
+
+
+
+</tr>
+
+</table>
+
+</td>
+
+                    <td>
+
+<table>
+
+<tr>
+
+<td><a href="/article/102.htm" title="自编招生通知2">自编招生通知2</a></td>
+
+
+
+<td>2026-09-9<div></div></td>
+
+
+
+</tr>
+
+</table>
+
+</td>
+
+                    </tr>

crawler/src/crawler/parse/fixtures/listing/trs.html

@@ -0,0 +1,16 @@
+<html>
+<head><meta charset="utf-8"></head>
+<body>
+<nav><a href="/contact.htm">自编导航</a></nav><main><ul class="clearfix">
+
+              <li>
+<a href="https://admission.example.edu.cn/notice/1">示例招生通知甲<i class="fa fa-angle-right"></i></a><span class="pull-right">[2026-09-10]</span>
+</li>
+<li>
+<a href="./202609/t20260909_102.html">示例招生通知乙<i class="fa fa-angle-right"></i></a><span class="pull-right">[2026-09-09]</span>
+</li>
+</ul>
+
+          </main>
+</body>
+</html>