From ab9e6d276c882e6fedb11ad69658a467013430d2 Mon Sep 17 00:00:00 2001 From: Tong Wu <141464206+w-tong2004@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:52:40 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E9=BD=90=E9=80=9A=E7=9F=A5=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E5=86=99=E6=B3=95=E5=B9=B6=E4=BF=9D=E7=95=99=E7=BC=96?= =?UTF-8?q?=E5=8F=B7=E8=AF=81=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/crawler/extraction/numbered_test.py | 14 +- crawler/src/crawler/parse/dates.py | 160 +++++++- crawler/src/crawler/parse/dates_test.py | 374 +++++++++++++++++- crawler/src/crawler/parse/indexing.py | 24 +- crawler/src/crawler/parse/indexing_test.py | 87 +++- docs/design.md | 7 +- 6 files changed, 635 insertions(+), 31 deletions(-) diff --git a/crawler/src/crawler/extraction/numbered_test.py b/crawler/src/crawler/extraction/numbered_test.py index 3b8ae40..cd56a4a 100644 --- a/crawler/src/crawler/extraction/numbered_test.py +++ b/crawler/src/crawler/extraction/numbered_test.py @@ -7,8 +7,18 @@ from crawler.parse.indexing import Document, index_documents -def test_selected_deadline_keeps_minutes_and_evidence_is_original_text() -> None: - text = "报名截止6月30日24时。" +@pytest.mark.parametrize( + "expression", + [ + "6月30日24时", + "2026-06-30 24:00", + "二\u3007二六年六月三十日24时", + "\uff086月30日\uff0924:00", + "6.30日24时", + ], +) +def test_selected_deadline_keeps_minutes_and_evidence_is_original_text(expression: str) -> None: + text = f"报名截止{expression}。" index = index_documents([Document(text)], published_on=date(2026, 5, 1)) prediction = resolve(selected_extraction(), index) field = prediction.programs[0].fields.apply diff --git a/crawler/src/crawler/parse/dates.py b/crawler/src/crawler/parse/dates.py index 18536ef..2c9aa1a 100644 --- a/crawler/src/crawler/parse/dates.py +++ b/crawler/src/crawler/parse/dates.py @@ -10,28 +10,55 @@ _CLOCK = r"(?:[0-9]{1,2}[:\uff1a][0-9]{2}|[0-9]{1,2}[时点](?:[0-9]{1,2}分?)?)" _WEEKDAY = r"[\uff08(]?(?:星期|周)[一二三四五六日天][\uff09)]?" -_GAP = rf"(?:[ \t]*(?:{_WEEKDAY}|上午|中午|下午|晚上))*[ \t]*" +_PERIOD = r"(?:上午|中午|下午|晚上)" +_GAP = rf"(?:[ \t]*(?:{_WEEKDAY}|{_PERIOD}))*[ \t]*" +_HAN_DIGITS = "零\u3007\u25cb一二三四五六七八九" +_DIGITS = dict(zip(_HAN_DIGITS, "000123456789", strict=True)) +_CHINESE_NUMBER = r"(?:[一二三四五六七八九]?十[一二三四五六七八九]?|[一二三四五六七八九])" +_NUMBER = rf"(?:[0-9]{{1,2}}|{_CHINESE_NUMBER})" +_YEAR = rf"(?:[0-9]{{4}}|[{_HAN_DIGITS}]{{4}})" +_MARK = r"(?:[ \t]*(?:前后|左右|之前|前))?" +_TIME = rf"(?:{_GAP}{_CLOCK}|[ \t]*止[ \t]*[\uff08(]{_CLOCK}[\uff09)])?" # Spaces are allowed between components, but a line break ends an expression. -_DAY = ( - rf"[0-9]{{1,2}}[ \t]*日(?:{_GAP}{_CLOCK}|[ \t]*止[ \t]*[\uff08(]{_CLOCK}[\uff09)])?" - r"(?:[ \t]*(?:前后|左右|之前|前))?" -) +_DAY = rf"{_NUMBER}[ \t]*[日号]{_TIME}{_MARK}" +_MONTH_START = rf"(?:{_YEAR}[ \t]*年[ \t]*)?{_NUMBER}[ \t]*月" _MONTH = ( - rf"(?:[0-9]{{4}}[ \t]*年[ \t]*)?[0-9]{{1,2}}[ \t]*月" + rf"{_MONTH_START}" rf"(?:[ \t]*(?:{_DAY}|中下旬|上旬|中旬|下旬|初|底))?" ) -_ATOM = rf"(?:即日起|{_MONTH})" +_NUMERIC_DATE = ( + r"(?:(?(?:[0-9]{{4}}[ \t]*年[ \t]*)?" - rf"[0-9]{{1,2}}[ \t]*月[ \t]*[0-9]{{1,2}}(?=[ \t]*{_SEPARATOR}[ \t]*{_DAY})|{_ATOM})" + rf"(?{_MONTH_START}[ \t]*{_NUMBER}" + rf"(?=[ \t]*{_SEPARATOR}[ \t]*(?:{_ATOM}|{_DAY}))|{_ATOM})" rf"(?:[ \t]*{_SEPARATOR}[ \t]*(?P{_ATOM}|{_DAY}|{_GAP}{_CLOCK}(?:前后|左右|之前|前)?))?" - r"(?!(?<=[0-9])[0-9])(?!(?<=月)[ \t]*[0-9])" + r"(?!(?<=[0-9])[0-9])(?!(?<=月)[ \t]*[0-9一二三四五六七八九十])" +) +_DATE_LABEL = re.compile( + r"(?:截止(?:时间|日期)?|(?:报名|申请|活动|考核|面试)(?:时间|日期)|日期|请于|定于)" + r"[ \t为是自从\uff1a:]*\Z" ) +_SHORT_DATE = re.compile(r"[0-9]{1,2}[ \t]*[-/.]") _POINT = re.compile( r"(?:(?P[0-9]{4})年)?(?:(?P[0-9]{1,2})月)?" r"(?:(?P[0-9]{1,2})日)?" - r"(?P上午|中午|下午|晚上)?" + rf"(?P{_PERIOD})?" r"(?:(?P[0-9]{1,2})[:\uff1a](?P[0-9]{2})|" r"(?P[0-9]{1,2})[时点](?:(?P[0-9]{1,2})分?)?)?" r"(?P上旬|中旬|下旬|初|底)?(?P前后|左右|之前|前)?" @@ -56,6 +83,80 @@ class DateMatch: issue: Literal["missing_publication", "invalid_date"] | None = None +def _chinese_number(match: re.Match[str]) -> str: + number = match[0] + if "十" not in number: + return "".join(_DIGITS[character] for character in number) + tens, _, units = number.partition("十") + return str(int(_DIGITS[tens] if tens else "1") * 10 + int(_DIGITS[units] if units else "0")) + + +def _numeric_date_context(match: re.Match[str], text: str) -> bool: + first, last = match["first"], match["last"] + parenthesized = first.startswith(("(", "\uff08")) + point = first[1:].lstrip(" \t") if parenthesized else first + first_is_numeric = re.match(_NUMERIC_DATE, point) is not None + last_is_short = last is not None and _SHORT_DATE.match(last) is not None + if not first_is_numeric and not last_is_short: + return True + before = text[: match.start()] + if first_is_numeric: + if re.search( + r"(?:版本(?:号)?|版|(? date: if isinstance(value, TimeToMonth): year, month = map(int, value.month.split("-")) @@ -79,9 +180,30 @@ def _point( previous: TimeValue | None = None, inherited_period: str | None = None, ) -> TimeValue | None: + # Add calendar units before removing delimiters so a clock digit cannot become part of the day. + numeric = re.match( + r"(?P[\uff08(][ \t]*)?" + r"(?:(?P[0-9]{4})[ \t]*(?P[-/.])[ \t]*)?" + r"(?P[0-9]{1,2})[ \t]*(?P[-/.])[ \t]*" + r"(?P[0-9]{1,2})[日号]?", + text, + ) + if numeric is not None: + if numeric["year"] and numeric["year_separator"] != numeric["day_separator"]: + raise ValueError("Mixed numeric date separators") + year = f"{numeric['year']}年" if numeric["year"] else "" + text = ( + f"{numeric['opening'] or ''}{year}{numeric['month']}月{numeric['day']}日" + f"{text[numeric.end() :]}" + ) compact = re.sub(r"[ \t]", "", text) compact = re.sub(_WEEKDAY, "", compact) compact = re.sub(rf"止[\uff08(]({_CLOCK})[\uff09)]$", r"\1", compact) + if re.fullmatch(rf"{_MONTH_START}{_NUMBER}", compact): + compact += "日" + compact = re.sub(rf"[{_HAN_DIGITS}十]+(?=[年月日号])", _chinese_number, compact) + compact = re.sub(rf"^[\uff08(]([^\uff09)]+)[\uff09)](?={_PERIOD}|[0-9])", r"\1", compact) + compact = compact.replace("号", "日") if compact == "即日起": return TimeToDay(date=published_on) if published_on is not None else None match = _POINT.fullmatch(compact) @@ -136,7 +258,12 @@ def _point( def find_dates(text: str, published_on: date | None) -> list[DateMatch]: matches: list[DateMatch] = [] - for match in _EXPRESSION.finditer(text): + position = 0 + while match := _EXPRESSION.search(text, position): + if not _numeric_date_context(match, text): + position = match.end("first") + continue + position = match.end() value: TimeValue | None = None end_value: TimeValue | None = None issue: Literal["missing_publication", "invalid_date"] | None = None @@ -147,19 +274,20 @@ def find_dates(text: str, published_on: date | None) -> list[DateMatch]: if last is None: last = first.removesuffix("中下旬") + "下旬" first = first.removesuffix("中下旬") + "中旬" - if first[-1].isdigit() and "月" in first and "日" not in first: - first += "日" value = _point(first, published_on) if value is None: issue = "missing_publication" if last is not None: - period = re.search(r"上午|中午|下午|晚上", first) + period = re.search(_PERIOD, first) end_value = _point(last, published_on, value, period[0] if period else None) if end_value is None: issue = "missing_publication" elif value is not None and _earliest(end_value) < _earliest(value): raise ValueError("Reversed date range") - if re.match(rf"[ \t]*{_SEPARATOR}[ \t]*[0-9]", text[match.end() :]): + if re.match( + rf"[ \t]*{_SEPARATOR}[ \t]*(?:[0-9]|[{_HAN_DIGITS}十百千万]+[ \t]*[年月日号])", + text[match.end() :], + ): issue = "invalid_date" if re.match(r"[ \t]*[0-9]{1,2}[:\uff1a][0-9]", text[match.end() :]): issue = "invalid_date" diff --git a/crawler/src/crawler/parse/dates_test.py b/crawler/src/crawler/parse/dates_test.py index 1ba5cd5..18a3c89 100644 --- a/crawler/src/crawler/parse/dates_test.py +++ b/crawler/src/crawler/parse/dates_test.py @@ -23,14 +23,17 @@ def test_dates_can_be_followed_immediately_by_numbers(text: str, days: list[int] @pytest.mark.parametrize( "text", [ - "6月1日-6月14号", + "6月1日-6月31号", "6月1日8:00-17:0", "6月1日8:00至17:001", "6月1日-6月14日17:0", "6月14日17:001", - "6月1日到14号", - "6月1日—14号", - "6月1日\uff5e14号", + "6月1日到31号", + "6月1日—31号", + "6月1日\uff5e31号", + "六月一日至零日", + "六月一日至一百日", + "六月一日至一百号", ], ) def test_unrecognized_numeric_range_end_cannot_leave_a_valid_start(text: str) -> None: @@ -46,6 +49,13 @@ def test_dash_before_prose_still_allows_an_ordinary_date() -> None: assert found.issue is None +def test_chinese_floor_after_a_date_is_not_an_invalid_range_endpoint() -> None: + [found] = find_dates("六月一日至一楼报到", date(2027, 5, 1)) + assert found.text == "六月一日" + assert found.value == TimeToDay(date=date(2027, 6, 1)) + assert found.issue is None + + def test_mid_to_late_month_does_not_replace_an_explicit_range_end() -> None: [found] = find_dates("7月中下旬至8月上旬", date(2027, 5, 1)) assert found.text == "7月中下旬至8月上旬" @@ -364,3 +374,359 @@ def test_before_a_clock_time_keeps_the_minute_value_and_original_qualifier( assert found.value.time == clock assert found.text == text assert found.issue is None + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("2027-06-30", TimeToDay(date=date(2027, 6, 30))), + ("2027/6/30", TimeToDay(date=date(2027, 6, 30))), + ("2027.6.30", TimeToDay(date=date(2027, 6, 30))), + ("2027 / 6 / 30 17:00", TimeToMinute(date=date(2027, 6, 30), time="17:00")), + ("2027-06-30下午5点前", TimeToMinute(date=date(2027, 6, 30), time="17:00")), + ("6月30号", TimeToDay(date=date(2027, 6, 30))), + ("6月30号24时", TimeToMinute(date=date(2027, 6, 30), time="24:00")), + ("二\u3007二七年八月五日", TimeToDay(date=date(2027, 8, 5))), + ("二零二七年十二月三十一号", TimeToDay(date=date(2027, 12, 31))), + ("二\u3007二八年二月二十九日", TimeToDay(date=date(2028, 2, 29))), + ("八月上旬", TimeToMonth(month="2027-08", part="early")), + ("九月中旬", TimeToMonth(month="2027-09", part="mid")), + ("十一月底", TimeToMonth(month="2027-11", part="late")), + ], +) +def test_additional_date_spellings_preserve_source_and_precision( + expression: str, expected: TimeValue +) -> None: + text = f"报名时间为{expression}\uff0c请及时提交。" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == expected + assert found.end_value is None + assert found.issue is None + assert found.text == expression + assert text[found.start : found.end] == expression + + +@pytest.mark.parametrize( + ("text", "expression", "expected"), + [ + ("报名截止为6.30", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("申请截止日期\uff1a6/30", "6/30", TimeToDay(date=date(2027, 6, 30))), + ("请于6.30前报名", "6.30前", TimeToDay(date=date(2027, 6, 30), mark="before")), + ("6.30日提交材料", "6.30日", TimeToDay(date=date(2027, 6, 30))), + ("报名截止6.30 17:00", "6.30 17:00", TimeToMinute(date=date(2027, 6, 30), time="17:00")), + ("6.30开始报名", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("6.30结束报名", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("6.30截止申请", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("6.30提交材料", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("6.30截止。", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("6.30截止\uff0c逾期不候。", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("6.30截止\n请提交材料。", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("6.30截止.", "6.30", TimeToDay(date=date(2027, 6, 30))), + ("通知称“6.30截止”。", "6.30", TimeToDay(date=date(2027, 6, 30))), + ], +) +def test_short_numeric_dates_need_adjacent_date_context( + text: str, expression: str, expected: TimeValue +) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert text[found.start : found.end] == expression + assert found.value == expected + assert found.issue is None + + +@pytest.mark.parametrize( + "text", + [ + "6.30", + "Python 3.14", + "版本6.30发布", + "版本v6.30前请完成报名", + "报名截止前升级到Python 3.14", + "IELTS≥6.0分", + "面积61.4万方", + "报名费6.30元", + "完成6/30的工作", + "报名时间使用版本6.30", + "温度3.14前后", + "价格6.30左右", + "2027-06-01至6.30元", + "2027-06-01至6/30分", + "版本2027.6.30", + "版本号\uff1a2027-06-30", + "version 2027/6/30", + "V 2027.6.30", + "版本\uff086.30\uff0917:00发布", + "版本(2027.6.30)17:00发布", + "版本( 2027.6.30 )17:00发布", + "版本\uff082027.6.30\uff09发布", + "版本2027 / 6 / 30报名", + "版本2027 - 6 - 30截止支持", + "版本2027. 6. 30截止支持", + "v1.2027.6.30", + "v1-2027-6-30", + "v1. 2027. 6. 30", + "v1 - 2027 - 6 - 30", + "v1 / 2027 / 6 / 30", + "一二三月", + "一二六月一日", + "6-30", + "数值区间1-10", + "报名费6-30元", + "报名截止\n6.30", + "从 Python 3.14开始支持新功能。", + "Python 3.14结束支持。", + "Python 3.14截止支持。", + "Python 3.14截止 支持。", + "6.30截止 支持。", + "版本号为6.30开始支持此接口。", + "版本号为6.30开始报名。", + "学分绩点3.20开始计算。", + "工作时间8.30-17:00。", + "面试时间8.30-17:00。", + "上午8.30开始报名。", + "下午6.30开始面试。", + "晚上8.30截止报名。", + "中午12.30开始考核。", + ], +) +def test_unrelated_numeric_expressions_are_not_promoted_to_dates(text: str) -> None: + assert find_dates(text, date(2027, 5, 1)) == [] + + +@pytest.mark.parametrize("prefix", ["1.", "2. ", "3.\t"]) +def test_numbered_list_items_do_not_hide_a_chinese_date(prefix: str) -> None: + text = prefix + "7月1日00:00至7月7日24:00报名。" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == "7月1日00:00至7月7日24:00" + assert found.value == TimeToMinute(date=date(2027, 7, 1), time="00:00") + assert found.end_value == TimeToMinute(date=date(2027, 7, 7), time="24:00") + + +@pytest.mark.parametrize( + ("text", "start", "end"), + [ + ("6月1日-6月14号", date(2027, 6, 1), date(2027, 6, 14)), + ("6月1号到14号", date(2027, 6, 1), date(2027, 6, 14)), + ("2027-06-01-2027-06-14", date(2027, 6, 1), date(2027, 6, 14)), + ("2027/6/1至6月14号", date(2027, 6, 1), date(2027, 6, 14)), + ("报名时间6.1至6.30", date(2027, 6, 1), date(2027, 6, 30)), + ("6.1-6.30日", date(2027, 6, 1), date(2027, 6, 30)), + ("6/1至6/30号", date(2027, 6, 1), date(2027, 6, 30)), + ("2027-06-01至6/30", date(2027, 6, 1), date(2027, 6, 30)), + ("2027-06-01至06-14", date(2027, 6, 1), date(2027, 6, 14)), + ("2027-06-30-07-01", date(2027, 6, 30), date(2027, 7, 1)), + ("二\u3007二七年十二月二十八日至一月十日", date(2027, 12, 28), date(2028, 1, 10)), + ("六月一至三十号", date(2027, 6, 1), date(2027, 6, 30)), + ("二\u3007二七年六月一至三十日", date(2027, 6, 1), date(2027, 6, 30)), + ("十二月二十八至一月十日", date(2027, 12, 28), date(2028, 1, 10)), + ("6月一至三十日", date(2027, 6, 1), date(2027, 6, 30)), + ], +) +def test_added_date_formats_share_range_and_year_rules(text: str, start: date, end: date) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == TimeToDay(date=start) + assert found.end_value == TimeToDay(date=end) + assert found.issue is None + + +@pytest.mark.parametrize( + ("text", "expected_day", "expected_time"), + [ + ("2027-6-1 17:00", 1, "17:00"), + ("2027 / 6 / 1\t08:30", 1, "08:30"), + ("2027.6.9 17点30分", 9, "17:30"), + ("报名截止6.1 08:30", 1, "08:30"), + ("报名截止6/9 17:00", 9, "17:00"), + ("报名截止6-1 24:00", 1, "24:00"), + ("(2027/6/1)17:00", 1, "17:00"), + ("\uff08 6.1 \uff09 08:30", 1, "08:30"), + ], +) +def test_numeric_day_does_not_absorb_the_first_clock_digit( + text: str, expected_day: int, expected_time: str +) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == TimeToMinute(date=date(2027, 6, expected_day), time=expected_time) + assert found.issue is None + assert text[found.start : found.end] == found.text + + +@pytest.mark.parametrize( + "text", + [ + "报名时间6.1 08:30至6.9 17:00", + "2027-6-1 08:30至2027-6-9 17:00", + "(2027/6/1)08:30至(2027/6/9)17:00", + ], +) +def test_numeric_range_endpoints_preserve_the_clock_boundary(text: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == TimeToMinute(date=date(2027, 6, 1), time="08:30") + assert found.end_value == TimeToMinute(date=date(2027, 6, 9), time="17:00") + assert found.issue is None + + +@pytest.mark.parametrize( + "activity", ["分别组织面试", "分批考核", "分为两组", "分组报到", "分阶段考核"] +) +def test_activity_verbs_are_not_mistaken_for_a_score_unit(activity: str) -> None: + [found] = find_dates(f"报名日期6.30{activity}", date(2027, 5, 1)) + assert found.value == TimeToDay(date=date(2027, 6, 30)) + assert found.issue is None + + +@pytest.mark.parametrize( + ("text", "end_day"), + [ + ("6.30号分批报名", None), + ("报名日期6月30日至7.1分别组织考核", 1), + ("报名日期6.30至7月1日分批考核", 1), + ], +) +def test_numeric_unit_filter_applies_only_to_an_ambiguous_range_endpoint( + text: str, end_day: int | None +) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == TimeToDay(date=date(2027, 6, 30)) + assert found.end_value == (TimeToDay(date=date(2027, 7, end_day)) if end_day else None) + assert found.issue is None + + +@pytest.mark.parametrize("text", ["6.30日8:00-17:00", "6.30 8:00-17:00", "报名日期6.30-17:00"]) +def test_short_date_clock_ranges_keep_explicit_calendar_context(text: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.issue is None + assert found.value == ( + TimeToDay(date=date(2027, 6, 30)) + if text.startswith("报名日期") + else TimeToMinute(date=date(2027, 6, 30), time="08:00") + ) + assert found.end_value == TimeToMinute(date=date(2027, 6, 30), time="17:00") + + +@pytest.mark.parametrize("text", ["报名截止6.30", "六月三十号", "八月上旬"]) +def test_new_yearless_formats_still_require_publication(text: str) -> None: + [found] = find_dates(text, None) + assert found.issue == "missing_publication" + assert found.value is None + assert found.end_value is None + + +@pytest.mark.parametrize( + "text", + [ + "2027-02-29", + "2027/6/31", + "二\u3007二七年二月二十九日", + "6月31号", + "2027-06-01至6/31", + "2027-06-30至2027-06-01", + "2027-06-30 24:01", + "2027/6-30", + "2027-06-01至06-31", + ], +) +def test_new_formats_preserve_invalid_date_and_range_failure(text: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.issue == "invalid_date" + assert found.value is None + assert found.end_value is None + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("\uff086月20日\uff0900:00", TimeToMinute(date=date(2027, 6, 20), time="00:00")), + ("(6月20日)08:30", TimeToMinute(date=date(2027, 6, 20), time="08:30")), + ( + "\uff08二\u3007二七年六月二十日\uff09下午2点前", + TimeToMinute(date=date(2027, 6, 20), time="14:00"), + ), + ("(2027/6/20) 24时", TimeToMinute(date=date(2027, 6, 20), time="24:00")), + ("( 2027/6/20 ) 24时", TimeToMinute(date=date(2027, 6, 20), time="24:00")), + ], +) +def test_parenthesized_dates_keep_a_clock_directly_after_the_closing_parenthesis( + expression: str, expected: TimeToMinute +) -> None: + text = f"报到安排为{expression}\uff0c请提前准备。" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert text[found.start : found.end] == expression + assert found.value == expected + assert found.issue is None + + +@pytest.mark.parametrize( + "text", + [ + "\uff086月20日\uff09\n08:30报到", + "6月20日\uff08注意事项\uff0908:30报到", + "\uff08见6月20日\uff0908:30报到", + ], +) +def test_clocks_are_not_joined_across_prose_or_line_breaks(text: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == "6月20日" + assert found.value == TimeToDay(date=date(2027, 6, 20)) + + +@pytest.mark.parametrize("weekday", ["周一", "星期一", "星期六"]) +@pytest.mark.parametrize("expression", ["6月1日17:00", "六月一日17:00"]) +def test_a_weekday_immediately_before_a_date_does_not_hide_it( + weekday: str, expression: str +) -> None: + text = f"{weekday}{expression}截止" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert found.value == TimeToMinute(date=date(2027, 6, 1), time="17:00") + + +@pytest.mark.parametrize("expression", ["2027-06-30", "2027/6/30", "2027.6.30"]) +@pytest.mark.parametrize("prefix", ["1.", "1. ", "1.\t", "1 . "]) +def test_numbered_lists_can_start_with_a_complete_numeric_date( + expression: str, prefix: str +) -> None: + text = f"{prefix}{expression}截止" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert text[found.start : found.end] == expression + assert found.value == TimeToDay(date=date(2027, 6, 30)) + + +@pytest.mark.parametrize("boundary", ["。", "\uff01", "?", ";", "\r", "\r\n"]) +@pytest.mark.parametrize("expression", ["2027-6-30", "6.30"]) +def test_numbered_dates_after_sentence_boundaries(boundary: str, expression: str) -> None: + text = f"说明{boundary}1. {expression}报名" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert found.value == TimeToDay(date=date(2027, 6, 30)) + assert found.issue is None + + +@pytest.mark.parametrize("expression", ["2027.6.30", "6.30日"]) +def test_a_version_label_on_the_previous_line_does_not_hide_a_date(expression: str) -> None: + text = f"版本\n{expression}截止" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert found.value == TimeToDay(date=date(2027, 6, 30)) + + +@pytest.mark.parametrize( + ("text", "expression", "expected"), + [ + ("版本6.30-7月1日发布", "7月1日", TimeToDay(date=date(2027, 7, 1))), + ("版本2027.6.30-7月1日发布", "7月1日", TimeToDay(date=date(2027, 7, 1))), + ("版本2027 / 6 / 30-7月1日发布", "7月1日", TimeToDay(date=date(2027, 7, 1))), + ("版本6.30至七月上旬更新", "七月上旬", TimeToMonth(month="2027-07", part="early")), + ], +) +def test_rejected_numeric_candidates_do_not_hide_a_later_date( + text: str, expression: str, expected: TimeValue +) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert text[found.start : found.end] == expression + assert found.value == expected + assert found.issue is None diff --git a/crawler/src/crawler/parse/indexing.py b/crawler/src/crawler/parse/indexing.py index 5a09c55..e0d7ca2 100644 --- a/crawler/src/crawler/parse/indexing.py +++ b/crawler/src/crawler/parse/indexing.py @@ -5,7 +5,7 @@ import re from collections.abc import Sequence -from dataclasses import dataclass, replace +from dataclasses import dataclass from datetime import date from typing import Literal from urllib.parse import unquote, urlsplit @@ -121,10 +121,13 @@ def _references(document: Document) -> list[_LocatedReference]: return sorted(references, key=lambda reference: reference.start) -def _sentences(text: str, references: list[_LocatedReference]) -> list[tuple[int, int]]: +def _sentences( + text: str, references: list[_LocatedReference], dates: list[DateMatch] +) -> list[tuple[int, int]]: spans: list[tuple[int, int]] = [] start = 0 reference_position = 0 + date_position = 0 for match in _BREAK.finditer(text): while ( reference_position < len(references) @@ -136,8 +139,12 @@ def _sentences(text: str, references: list[_LocatedReference]) -> list[tuple[int and references[reference_position].start < match.end() ): continue - if match[0] == "." and re.fullmatch( - r"[0-9]+(?:\.[0-9]+)*", text[start : match.start()].strip() + while date_position < len(dates) and dates[date_position].end <= match.start(): + date_position += 1 + # A dot inside a spaced numeric date must not truncate its evidence sentence. + if match[0] == "." and ( + (date_position < len(dates) and dates[date_position].start <= match.start()) + or re.fullmatch(r"[0-9]+(?:\.[0-9]+)*", text[start : match.start()].strip()) ): continue newline = match[0][0] in "\r\n" @@ -163,8 +170,10 @@ def index_documents(documents: Sequence[Document], *, published_on: date | None) index = NoticeIndex({}, {}, {}, {}) for document in documents: references = _references(document) + dates = find_dates(document.text, published_on) reference_position = 0 - for start, end in _sentences(document.text, references): + date_position = 0 + for start, end in _sentences(document.text, references, dates): sentence_id = f"S{len(index.sentences) + 1}" sentence = Sentence( sentence_id, document.attachment, start, end, document.text[start:end] @@ -189,8 +198,9 @@ def index_documents(documents: Sequence[Document], *, published_on: date | None) (start + address.start(), start + address.end()) for address in _REFERENCE.finditer(sentence.text) ] - for match in find_dates(sentence.text, published_on): - located = replace(match, start=start + match.start, end=start + match.end) + while date_position < len(dates) and dates[date_position].start < end: + located = dates[date_position] + date_position += 1 # Dates embedded in a URL or an email are not dates asserted by the notice. if any( located.start < address_end and located.end > address_start diff --git a/crawler/src/crawler/parse/indexing_test.py b/crawler/src/crawler/parse/indexing_test.py index 9bd49aa..9efa51e 100644 --- a/crawler/src/crawler/parse/indexing_test.py +++ b/crawler/src/crawler/parse/indexing_test.py @@ -2,7 +2,7 @@ import pytest -from crawler.crawl.contract import TimeToDay, TimeToMinute +from crawler.crawl.contract import TimeToDay, TimeToMinute, TimeToMonth from crawler.parse.indexing import Document, Hyperlink, index_documents PUBLISHED = date(2027, 6, 1) @@ -128,3 +128,88 @@ def test_dates_in_addresses_are_excluded_but_anchor_labels_are_kept() -> None: published_on=PUBLISHED, ) assert [d.match.text for d in index.dates.values()] == ["6月18日"] + + +def test_extended_dates_preserve_document_offsets_and_exclude_numeric_addresses() -> None: + body = ( + "https://example.org/2028-06-30。camp2028.06.30@example.org。\r\n" + "报名截止2028/6/30 17\uff1a00。" + ) + label = "2028/6/30 17\uff1a00" + start = body.index(label) + documents = [ + Document( + body, hyperlinks=(Hyperlink(start, start + len(label), "https://apply.example.org"),) + ), + Document("七月上旬举办。\r\n报名截止7.5前。", attachment=0), + ] + index = index_documents(documents, published_on=date(2028, 6, 1)) + mentions = list(index.dates.values()) + assert [m.match.text for m in mentions] == [label, "七月上旬", "7.5前"] + assert [m.match.value for m in mentions] == [ + TimeToMinute(date=date(2028, 6, 30), time="17:00"), + TimeToMonth(month="2028-07", part="early"), + TimeToDay(date=date(2028, 7, 5), mark="before"), + ] + originals = {document.attachment: document.text for document in documents} + for mention in mentions: + sentence = index.sentences[mention.sentence_id] + assert ( + originals[sentence.attachment][mention.match.start : mention.match.end] + == mention.match.text + ) + assert index.evidence(mentions[-1].sentence_id).attachment == 0 + + +@pytest.mark.parametrize( + ("expression", "start", "end"), + [ + ("2027. 6. 30 17:00", TimeToMinute(date=date(2027, 6, 30), time="17:00"), None), + ("6. 30", TimeToDay(date=date(2027, 6, 30)), None), + ("6. 30至7. 1", TimeToDay(date=date(2027, 6, 30)), TimeToDay(date=date(2027, 7, 1))), + ], +) +def test_spaced_numeric_dates_stay_in_their_evidence_sentence( + expression: str, start: TimeToDay | TimeToMinute, end: TimeToDay | None +) -> None: + sentence = f"报名时间{expression}。" + text = f"Intro. {sentence}Done." + index = index_documents([Document(text)], published_on=PUBLISHED) + assert [s.text for s in index.sentences.values()] == ["Intro.", sentence, "Done."] + [mention] = index.dates.values() + assert mention.match.text == expression + assert mention.match.value == start + assert mention.match.end_value == end + assert text[mention.match.start : mention.match.end] == expression + assert index.evidence(mention.sentence_id).quote == sentence + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("版本6.30-7月1日发布", "7月1日"), + ("版本2027.6.30-7月1日发布", "7月1日"), + ("版本6.30至七月上旬更新", "七月上旬"), + ], +) +def test_rejected_numeric_versions_do_not_hide_later_date_ids(text: str, expected: str) -> None: + index = index_documents([Document(text)], published_on=PUBLISHED) + [mention] = index.dates.values() + assert mention.match.text == expected + assert mention.match.issue is None + assert text[mention.match.start : mention.match.end] == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("版本2027. 6. 30截止支持", []), + ("版本2027. 6. 30-7月1日发布", ["7月1日"]), + ("版本2027. 6. 30发布。报名截止7. 1。", ["7. 1"]), + ], +) +def test_sentence_boundaries_keep_the_full_numeric_rejection_context( + text: str, expected: list[str] +) -> None: + index = index_documents([Document(text)], published_on=PUBLISHED) + assert [mention.match.text for mention in index.dates.values()] == expected diff --git a/docs/design.md b/docs/design.md index ed64310..8252fda 100644 --- a/docs/design.md +++ b/docs/design.md @@ -250,7 +250,12 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 | 7 月上旬、中旬、下旬；月初算上旬，月底算下旬 | `{"month": "2026-07", "part": "early"}`，`mid`、`late` | | 7 月 | `{"month": "2026-07"}` | -- 没写年份的先按发布年份算，比发布日期早半年以上的算下一年：12 月发的通知写「1 月 10 日截止」算下一年，回顾 7 月的夏令营仍算当年。时间段右端没写的年月跟左端，跨年同样按这条。「即日起」是发布那天；发布日期为空时，要靠它推年份的不给时间值。 +- 日期的数字可以用阿拉伯数字或中文数字，比如「二〇二六年六月三十日」「九月中旬」。「号」和「日」等价。完整年月日也接受 `2026-06-30`、`2026/6/30`、`2026.6.30`，同一日期的数字分隔符一致。数字日期也能带时刻，比如 `2026-6-1 17:00`。 +- 日期在括号内、时刻紧跟在括号外时一起识别，比如「（6 月 30 日）17:00」。日期在全文识别一次，编号分句复用这些匹配并保护日期内部的句点，原文和字符位置保留不变。 +- `6.30`、`6/30`、`6-30` 这样的短数字月日，要有紧邻的日期标签（如「报名时间」「截止日期」「请于」）、日或号的后缀、报名或材料提交等具体活动动作，或者处在另一端有明确日期的同一范围里。括号包住短日期并紧跟时刻也提供日期上下文。 +- 短数字后的「开始」「结束」需要连接具体活动。「截止」后为句末或标点时也提供日期线索，「截止支持」不提供这条线索。单独的短数字不猜成日期。数字日期前紧邻版本标识时排除。数值单位用于排除紧邻它、且没有日/号的短数字端点，「分别」「分批」等活动动词提供的上下文照常判断。前面紧邻「上午」「中午」「下午」「晚上」、且没有日/号的短数字按时刻文字保留。 +- 短数字后只接另一个时刻的范围（如 `8.30-17:00`）有日期和日内时间两种理解，需要日/号、含「日期」的标签或首端独立的时刻才能按日期范围识别。识别在同一行内进行，不把表格跨行的日期和时刻自动拼接。 +- 日期范围的首端可省略日/号，比如「六月一至三十日」「十二月二十八至一月十日」。没写年份的先按发布年份算，比发布日期早半年以上的算下一年：12 月发的通知写「1 月 10 日截止」算下一年，回顾 7 月的夏令营仍算当年。时间段右端没写的年月跟左端，跨年同样按这条。「即日起」是发布那天；发布日期为空时，要靠它推年份的不给时间值。 - 换算成时刻（排序、倒计时、状态和提醒用）时，拿不准的往早里算：只写到日的截止算到当天结束，带 `mark` 的算到前一天结束，只写到旬或月的算到这段时间开始；开始都从最早的可能算起。页面上照原文的写法显示，不显示换算出的时刻。 ### 8.3 校验