From c72bb90dcc153379286feb76c0d813867dd37f05 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Sun, 28 Jun 2026 18:08:02 +0200 Subject: [PATCH 1/3] maintenance: remove babel dependency --- courlan/filters.py | 23 ++-- courlan/langcodes.py | 313 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 325 insertions(+), 11 deletions(-) create mode 100644 courlan/langcodes.py diff --git a/courlan/filters.py b/courlan/filters.py index 5fcf0c5e..8c5a53af 100644 --- a/courlan/filters.py +++ b/courlan/filters.py @@ -8,7 +8,7 @@ from ipaddress import ip_address from urllib.parse import SplitResult, urlsplit -from babel import Locale, UnknownLocaleError +from .langcodes import ISO_LANGS, ISO_TERRS LOGGER = logging.getLogger(__name__) @@ -179,16 +179,17 @@ def extension_filter(urlpath: str) -> bool: @lru_cache(maxsize=1024) def langcodes_score(language: str, segment: str, score: int) -> int: - "Use locale parser to assess the plausibility of the chosen URL segment." + "Assess the plausibility of a URL segment as a language indicator." + if not isinstance(segment, str): + return score delimiter = "_" if "_" in segment else "-" - try: - if Locale.parse(segment, sep=delimiter).language == language: - score += 1 - else: - score -= 1 - except (TypeError, UnknownLocaleError): - pass - return score + lang, _, territory = segment.partition(delimiter) + lang = lang.lower() + if lang not in ISO_LANGS: + return score + if territory and territory.upper() not in ISO_TERRS: + return score + return score + (1 if lang == language else -1) def lang_filter( @@ -215,7 +216,7 @@ def lang_filter( score = langcodes_score(language, match[1], score) elif len(occurrences) == 2: for occurrence in occurrences: - score = langcodes_score(language, occurrence, score) + score = langcodes_score(language, occurrence[0] + occurrence[1], score) # don't perform the test if there are too many candidates: > 2 # second test: prepended language cues if strict: diff --git a/courlan/langcodes.py b/courlan/langcodes.py new file mode 100644 index 00000000..7d408acf --- /dev/null +++ b/courlan/langcodes.py @@ -0,0 +1,313 @@ +""" +Language and territory code sets derived from CLDR via babel 2.18, used for +URL language detection. Replaces the babel runtime dependency. +""" + +# ISO 639-1 language codes accepted by CLDR (167 codes). +# Generated by: all ab where Locale.parse(ab) succeeds in babel 2.18. +ISO_LANGS = frozenset( + ( + "aa", + "ab", + "af", + "ak", + "am", + "an", + "ar", + "as", + "az", + "ba", + "be", + "bg", + "bh", + "bm", + "bn", + "bo", + "br", + "bs", + "ca", + "ce", + "co", + "cs", + "cu", + "cv", + "cy", + "da", + "de", + "dv", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "ff", + "fi", + "fo", + "fr", + "fy", + "ga", + "gd", + "gl", + "gn", + "gu", + "gv", + "ha", + "he", + "hi", + "hr", + "ht", + "hu", + "hy", + "ia", + "id", + "ie", + "ig", + "ii", + "in", + "io", + "is", + "it", + "iu", + "iw", + "ja", + "ji", + "jv", + "jw", + "ka", + "ki", + "kk", + "kl", + "km", + "kn", + "ko", + "ks", + "ku", + "kw", + "ky", + "la", + "lb", + "lg", + "ln", + "lo", + "lt", + "lu", + "lv", + "mg", + "mi", + "mk", + "ml", + "mn", + "mo", + "mr", + "ms", + "mt", + "my", + "nb", + "nd", + "ne", + "nl", + "nn", + "no", + "nr", + "nv", + "ny", + "oc", + "om", + "or", + "os", + "pa", + "pl", + "ps", + "pt", + "qu", + "rm", + "rn", + "ro", + "ru", + "rw", + "sa", + "sc", + "sd", + "se", + "sg", + "si", + "sk", + "sl", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "ta", + "te", + "tg", + "th", + "ti", + "tk", + "tl", + "tn", + "to", + "tr", + "ts", + "tt", + "tw", + "ug", + "uk", + "ur", + "uz", + "ve", + "vi", + "vo", + "wa", + "wo", + "xh", + "yi", + "yo", + "za", + "zh", + "zu", + ) +) + +# ISO 3166-1 alpha-2 territory codes accepted by CLDR (128 codes, incl. historical). +# Generated by: all XX where Locale.parse("en_XX") succeeds in babel 2.18. +ISO_TERRS = frozenset( + ( + "AE", + "AG", + "AI", + "AS", + "AT", + "AU", + "BB", + "BE", + "BI", + "BM", + "BS", + "BW", + "BZ", + "CA", + "CC", + "CH", + "CK", + "CM", + "CT", + "CX", + "CY", + "CZ", + "DD", + "DE", + "DG", + "DK", + "DM", + "ER", + "ES", + "FI", + "FJ", + "FK", + "FM", + "FR", + "FX", + "GB", + "GD", + "GG", + "GH", + "GI", + "GM", + "GS", + "GU", + "GY", + "HK", + "HU", + "ID", + "IE", + "IL", + "IM", + "IN", + "IO", + "IT", + "JE", + "JM", + "JT", + "KE", + "KI", + "KN", + "KY", + "LC", + "LR", + "LS", + "MG", + "MH", + "MI", + "MO", + "MP", + "MS", + "MT", + "MU", + "MV", + "MW", + "MY", + "NA", + "NF", + "NG", + "NH", + "NL", + "NO", + "NR", + "NU", + "NZ", + "PC", + "PG", + "PH", + "PK", + "PL", + "PN", + "PR", + "PT", + "PU", + "PW", + "RH", + "RO", + "RW", + "SB", + "SC", + "SD", + "SE", + "SG", + "SH", + "SI", + "SK", + "SL", + "SS", + "SX", + "SZ", + "TC", + "TK", + "TO", + "TT", + "TV", + "TZ", + "UG", + "UK", + "UM", + "US", + "VC", + "VG", + "VI", + "VU", + "WK", + "WS", + "ZA", + "ZM", + "ZW", + "ZZ", + ) +) From 09ca2167132b6b7c4e35be549a497d482dd741e7 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Tue, 30 Jun 2026 21:47:13 +0200 Subject: [PATCH 2/3] better filter coverage --- courlan/filters.py | 18 +++++++++--------- pyproject.toml | 3 +-- tests/unit_tests.py | 24 +++++++++++++++++------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/courlan/filters.py b/courlan/filters.py index 8c5a53af..0f64f811 100644 --- a/courlan/filters.py +++ b/courlan/filters.py @@ -79,7 +79,7 @@ # language filter PATH_LANG_FILTER = re.compile( - r"(?:https?://[^/]+/)([a-z]{2})([_-][a-z]{2,3})?(?:/|$)", re.IGNORECASE + r"(?:https?://[^/]+/)([a-z]{2})([_-][a-z]{2})?(?:/|$)", re.IGNORECASE ) ALL_PATH_LANGS = re.compile(r"(?:/)([a-z]{2})([_-][a-z]{2})?(?:/)", re.IGNORECASE) ALL_PATH_LANGS_NO_TRAILING = re.compile( @@ -178,18 +178,18 @@ def extension_filter(urlpath: str) -> bool: @lru_cache(maxsize=1024) -def langcodes_score(language: str, segment: str, score: int) -> int: - "Assess the plausibility of a URL segment as a language indicator." +def langcodes_score(language: str, segment: str) -> int: + "Score a URL segment as a language indicator: +1 if it is a plausible matching locale, -1 if a mismatching one, 0 if not a recognizable locale." if not isinstance(segment, str): - return score + return 0 delimiter = "_" if "_" in segment else "-" lang, _, territory = segment.partition(delimiter) lang = lang.lower() if lang not in ISO_LANGS: - return score + return 0 if territory and territory.upper() not in ISO_TERRS: - return score - return score + (1 if lang == language else -1) + return 0 + return 1 if lang == language else -1 def lang_filter( @@ -213,10 +213,10 @@ def lang_filter( else: occurrences = ALL_PATH_LANGS_NO_TRAILING.findall(url) if len(occurrences) == 1: - score = langcodes_score(language, match[1], score) + score += langcodes_score(language, match[1] + (match[2] or "")) elif len(occurrences) == 2: for occurrence in occurrences: - score = langcodes_score(language, occurrence[0] + occurrence[1], score) + score += langcodes_score(language, occurrence[0] + occurrence[1]) # don't perform the test if there are too many candidates: > 2 # second test: prepended language cues if strict: diff --git a/pyproject.toml b/pyproject.toml index b08676fe..d1cbb402 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ license = "Apache-2.0" dynamic = ["version"] requires-python = ">=3.10" authors = [ - {name = "Adrien Barbaresi", email = "barbaresi@bbaw.de"} + {name = "Adrien Barbaresi"} ] keywords=[ "cleaner", @@ -49,7 +49,6 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "babel >= 2.16.0", "tld >= 0.13", "urllib3 >= 1.26, < 3", ] diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 6fafc4c0..f581cec3 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -468,18 +468,28 @@ def test_lang_filter(): lang_filter("http://bz.berlin1.de/kino/050513/fans.html", "de", strict=True) is False ) - assert langcodes_score("en", "en_HK", 0) == 1 - assert langcodes_score("en", "en-HK", 0) == 1 - assert langcodes_score("en", "en_XY", 0) == 0 - assert langcodes_score("en", "en-XY", 0) == 0 - assert langcodes_score("en", "de_DE", 0) == -1 - assert langcodes_score("en", "de-DE", 0) == -1 + # both path segments differ from target — was always True when two-occurrence branch was dead + assert lang_filter("https://x.com/fr/x/de/", "en") is False + # invalid territory (en_XY) is not a confident match, so the de segment wins (0, -1 → -1) + assert lang_filter("https://x.com/en_XY/x/de/", "en") is False # assert lang_filter('http://www.verfassungen.de/ch/basel/verf03.htm'. 'de') is True # assert lang_filter('http://www.uni-stuttgart.de/hi/fnz/lehrveranst.html', 'de') is True # http://www.wildwechsel.de/ww/front_content.php?idcatart=177&lang=4&client=6&a=view&eintrag=100&a=view&eintrag=0&a=view&eintrag=20&a=view&eintrag=80&a=view&eintrag=20 +def test_langcodes_score(): + assert langcodes_score("en", "en_HK") == 1 + assert langcodes_score("en", "en-HK") == 1 + assert langcodes_score("en", "de_DE") == -1 + assert langcodes_score("en", "de-DE") == -1 + assert langcodes_score("en", "en_XY") == 0 # invalid territory + assert langcodes_score("en", "en-XY") == 0 + assert langcodes_score("en", "xx") == 0 # invalid language + assert langcodes_score("en", "xx_US") == 0 + assert langcodes_score("en", None) == 0 # non-string input + + def test_navigation(): assert is_navigation_page("https://test.org/") is False assert is_navigation_page("https://test.org/page/1") is True @@ -1379,7 +1389,7 @@ def test_examples(): def test_meta(): "Test package meta functions." - _ = langcodes_score("en", "en_HK", 0) + _ = langcodes_score("en", "en_HK") _ = _parse("https://example.net/123/abc") # urlsplit is only an lru_cache wrapper on some Python versions From 745364badfb74baee58455a0582c6c1199b5bb27 Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Tue, 30 Jun 2026 22:15:36 +0200 Subject: [PATCH 3/3] fix indirect coverage --- tests/unit_tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index f581cec3..4ca5120a 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -472,8 +472,11 @@ def test_lang_filter(): assert lang_filter("https://x.com/fr/x/de/", "en") is False # invalid territory (en_XY) is not a confident match, so the de segment wins (0, -1 → -1) assert lang_filter("https://x.com/en_XY/x/de/", "en") is False + # >2 candidates: unreliable, score stays 0 + assert lang_filter("https://x.com/en/x/de/x/fr/", "en") is True - # assert lang_filter('http://www.verfassungen.de/ch/basel/verf03.htm'. 'de') is True + # /ch/ is not a language code, no rejection + assert lang_filter("http://www.verfassungen.de/ch/basel/verf03.htm", "de") is True # assert lang_filter('http://www.uni-stuttgart.de/hi/fnz/lehrveranst.html', 'de') is True # http://www.wildwechsel.de/ww/front_content.php?idcatart=177&lang=4&client=6&a=view&eintrag=100&a=view&eintrag=0&a=view&eintrag=20&a=view&eintrag=80&a=view&eintrag=20