From 57d24a0d399656a662623bc6ec4e7b10e274a998 Mon Sep 17 00:00:00 2001 From: JunieXD <1240840076@qq.com> Date: Mon, 9 Nov 2026 22:01:38 +0800 Subject: [PATCH] =?UTF-8?q?=E9=A2=86=E6=B4=BB=E4=B8=8D=E5=86=8D=E5=B8=A6?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=E7=9A=84=E6=A0=8F=E7=9B=AE=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 初筛只看标题（总体设计 §8.1），抓取节点用不到栏目名；网站上它还是来源的名字，列留着。 - 领活接口的 WorkSource 去掉 section，重新生成 OpenAPI 文档和抓取节点的类型 - 数据表文档改掉「初筛时一起看」的说法 --- crawler/src/crawler/crawl/browser_test.py | 4 +--- crawler/src/crawler/crawl/contract.py | 1 - crawler/src/crawler/crawl/notice_test.py | 1 - crawler/src/crawler/crawl/rounds_test.py | 4 +--- crawler/src/crawler/parse/listing_test.py | 4 +--- crawler/src/crawler/screening/screen_test.py | 1 - docs/data-model.md | 2 +- web/openapi/crawler.json | 4 ---- web/server/modules/ingest/routes.ts | 1 - web/server/modules/sources/service.ts | 2 +- 10 files changed, 5 insertions(+), 19 deletions(-) diff --git a/crawler/src/crawler/crawl/browser_test.py b/crawler/src/crawler/crawl/browser_test.py index 644a5fb..98d26be 100644 --- a/crawler/src/crawler/crawl/browser_test.py +++ b/crawler/src/crawler/crawl/browser_test.py @@ -10,9 +10,7 @@ from crawler.crawl.contract import WorkSource from crawler.crawl.fetch import INTERVAL, Fetcher, FetchError -SOURCE = WorkSource( - id=7, url="https://a/list", section="通知公告", browser=True, config=None, backfill=False -) +SOURCE = WorkSource(id=7, url="https://a/list", browser=True, config=None, backfill=False) def run(main: Callable[[], Awaitable[None]]) -> None: diff --git a/crawler/src/crawler/crawl/contract.py b/crawler/src/crawler/crawl/contract.py index 925d454..f4f1868 100644 --- a/crawler/src/crawler/crawl/contract.py +++ b/crawler/src/crawler/crawl/contract.py @@ -162,7 +162,6 @@ class ProbeReport(BaseModel): class WorkSource(BaseModel): id: Annotated[int, Field(ge=-2147483648, le=2147483647)] url: str - section: str browser: bool config: Config | None backfill: Annotated[bool, Field(description="第一次抓：往回翻页，补上前两年的通知")] diff --git a/crawler/src/crawler/crawl/notice_test.py b/crawler/src/crawler/crawl/notice_test.py index 496831b..6a7c695 100644 --- a/crawler/src/crawler/crawl/notice_test.py +++ b/crawler/src/crawler/crawl/notice_test.py @@ -13,7 +13,6 @@ SOURCE = WorkSource( id=7, url="https://example.edu.cn/tzgg.htm", - section="通知公告", browser=False, config=None, backfill=False, diff --git a/crawler/src/crawler/crawl/rounds_test.py b/crawler/src/crawler/crawl/rounds_test.py index e78635b..dd7d203 100644 --- a/crawler/src/crawler/crawl/rounds_test.py +++ b/crawler/src/crawler/crawl/rounds_test.py @@ -21,9 +21,7 @@ from crawler.crawl.rounds import Attachment, NoticePage, StepError, Steps, crawl_source from crawler.crawl.website import Website -SOURCE = WorkSource( - id=7, url="http://a/list.htm", section="通知公告", browser=False, config=None, backfill=False -) +SOURCE = WorkSource(id=7, url="http://a/list.htm", browser=False, config=None, backfill=False) ITEMS = [ListedItem(url=f"http://a/{n}.htm", title=f"通知 {n}") for n in range(1, 5)] CONTEXT = ExtractionContext(published_on=None, colleges=[], programs=[]) USAGE = [Usage(model="m", input_tokens=10, output_tokens=1)] diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py index 98c2eb8..c1a61ce 100644 --- a/crawler/src/crawler/parse/listing_test.py +++ b/crawler/src/crawler/parse/listing_test.py @@ -21,9 +21,7 @@ def source(*, backfill: bool = False, config: dict[str, JsonValue] | None = None) -> WorkSource: - return WorkSource( - id=7, url=BASE, section="招生信息", browser=False, config=config, backfill=backfill - ) + return WorkSource(id=7, url=BASE, browser=False, config=config, backfill=backfill) def page(rows: list[tuple[str, str | None]], following: str | None = None) -> bytes: diff --git a/crawler/src/crawler/screening/screen_test.py b/crawler/src/crawler/screening/screen_test.py index c16090c..9af3cb0 100644 --- a/crawler/src/crawler/screening/screen_test.py +++ b/crawler/src/crawler/screening/screen_test.py @@ -13,7 +13,6 @@ def source() -> WorkSource: return WorkSource( id=7, url="https://example.edu.cn", - section="招生", browser=False, config=None, backfill=False, diff --git a/docs/data-model.md b/docs/data-model.md index a42a9d2..ccb1090 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -26,7 +26,7 @@ | `schools` | `sources` | 学校：名称、简称（可以有几个，只用来搜索）、层次（`c9`、`985`、`211`、`double_first_class` 中最高的一个，没有为空） | | `school_logos` | `sources` | 学校的校徽：图片、类型和修改时间。页面从鼠钟取校徽，不去学校的网站拿 | | `colleges` | `sources` | 学院（泛指招生单位，`kind` 是学院、研究所或研究生院），院校清单就是这张表。`region` 是学生就读的地区：深圳校区、中科院的研究所和学校不在一地 | -| `sources` | `sources` | 一个通知列表页。`section` 是网站上的栏目名，初筛时一起看；`browser` 为真要用浏览器抓；`config` 是抓取节点的配置，格式由抓取节点定，为空就用通用抓取；`unavailable_reason` 有值就是暂时抓不了，不再抓。`crawled_at` 是上一轮结束的时间，为空说明还没抓过，第一轮要往回翻页补上往年的；`leased_until` 见抓取节点接口的租约 | +| `sources` | `sources` | 一个通知列表页。`section` 是网站上的栏目名，页面上拿它称呼这个来源（抓取看板、学院页、来源管理），抓取节点用不到；`browser` 为真要用浏览器抓；`config` 是抓取节点的配置，格式由抓取节点定，为空就用通用抓取；`unavailable_reason` 有值就是暂时抓不了，不再抓。`crawled_at` 是上一轮结束的时间，为空说明还没抓过，第一轮要往回翻页补上往年的；`leased_until` 见抓取节点接口的租约 | | `source_probes` | `sources` | 管理员要试抓的地址，和抓取节点交回的条目、建议的配置或失败原因；`finished_at` 为空就是还在等 | ## 抓取 diff --git a/web/openapi/crawler.json b/web/openapi/crawler.json index 0ed5c1c..e56255c 100644 --- a/web/openapi/crawler.json +++ b/web/openapi/crawler.json @@ -49,9 +49,6 @@ "url": { "type": "string" }, - "section": { - "type": "string" - }, "browser": { "type": "boolean" }, @@ -73,7 +70,6 @@ "required": [ "id", "url", - "section", "browser", "config", "backfill" diff --git a/web/server/modules/ingest/routes.ts b/web/server/modules/ingest/routes.ts index 358755b..9d70331 100644 --- a/web/server/modules/ingest/routes.ts +++ b/web/server/modules/ingest/routes.ts @@ -89,7 +89,6 @@ const Work = z .object({ id: Int, url: z.string(), - section: z.string(), browser: z.boolean(), config: z.union([Config, z.null()]), backfill: z.boolean().openapi({ description: "第一次抓：往回翻页，补上前两年的通知" }), diff --git a/web/server/modules/sources/service.ts b/web/server/modules/sources/service.ts index b40bc4e..edd627d 100644 --- a/web/server/modules/sources/service.ts +++ b/web/server/modules/sources/service.ts @@ -566,7 +566,7 @@ export async function leaseSources( .forUpdate() .skipLocked(), ) - .returning(["id", "url", "section", "browser", "crawled_at"]) + .returning(["id", "url", "browser", "crawled_at"]) .returning(sql | null>`config`.as("config")) .execute(); }