diff --git a/app/reviews/autoreview/checks/__init__.py b/app/reviews/autoreview/checks/__init__.py index db35eeef..feac348b 100644 --- a/app/reviews/autoreview/checks/__init__.py +++ b/app/reviews/autoreview/checks/__init__.py @@ -8,6 +8,7 @@ from .invalid_isbn import check_invalid_isbn from .manual_unapproval import check_manual_unapproval from .ores_scores import check_ores_scores +from .reference_only_edit import check_reference_only_edit from .render_errors import check_render_errors from .superseded_additions import check_superseded_additions from .user_block import check_user_block @@ -67,6 +68,12 @@ "function": check_invalid_isbn, "priority": 8, }, + { + "id": "reference-only-edit", + "name": "Reference-only edit", + "function": check_reference_only_edit, + "priority": 8.5, + }, { "id": "superseded-additions", "name": "Superseded additions", diff --git a/app/reviews/autoreview/checks/reference_only_edit.py b/app/reviews/autoreview/checks/reference_only_edit.py new file mode 100644 index 00000000..01eb3589 --- /dev/null +++ b/app/reviews/autoreview/checks/reference_only_edit.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import logging + +from ..base import CheckResult +from ..context import CheckContext +from ..decision import AutoreviewDecision +from ..utils.wikitext import ( + extract_domain_from_url, + extract_urls_from_references, + get_parent_wikitext, + is_reference_only_edit, +) + +logger = logging.getLogger(__name__) + + +def check_reference_only_edit(context: CheckContext) -> CheckResult: + """Check if revision only adds or modifies references.""" + pending_wikitext = context.revision.get_wikitext() + parent_wikitext = get_parent_wikitext(context.revision) + + is_ref_only, has_removals, new_or_modified_refs = is_reference_only_edit( + parent_wikitext, pending_wikitext + ) + + if not is_ref_only: + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="skip", + message="Edit modifies content beyond references.", + ) + + if has_removals and not new_or_modified_refs: + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="not_ok", + message="Edit only removes references without adding new ones.", + decision=AutoreviewDecision( + status="manual", + label="Requires manual review", + reason="Reference-only edits that only remove references require manual review.", + ), + should_stop=True, + ) + + if not new_or_modified_refs: + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="skip", + message="No new or modified references detected.", + ) + + urls = extract_urls_from_references(new_or_modified_refs) + + if not urls: + logger.info( + "Auto-approving reference-only edit %s (no URLs in new references)", + context.revision.revid, + ) + ref_count = len(new_or_modified_refs) + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="ok", + message=f"Edit only modifies references ({ref_count} reference(s) added/modified).", + decision=AutoreviewDecision( + status="approve", + label="Can be auto-approved", + reason="Edit only adds or modifies references without external URLs.", + ), + should_stop=True, + ) + + domains = [] + for url in urls: + domain = extract_domain_from_url(url) + if domain: + domains.append(domain) + + new_domains = [] + checked_domains = set() + + # Get the page ID to exclude from domain usage search (avoid self-matches) + page_id = context.revision.page.pageid if hasattr(context.revision.page, "pageid") else None + + for domain in domains: + if domain in checked_domains: + continue + checked_domains.add(domain) + + has_been_used = context.client.has_domain_been_used(domain, exclude_page_id=page_id) + + if not has_been_used: + new_domains.append(domain) + logger.info( + "Domain %s has not been used before in revision %s", + domain, + context.revision.revid, + ) + + if new_domains: + domain_list = ", ".join(new_domains[:3]) + if len(new_domains) > 3: + domain_list += "..." + domain_count = len(new_domains) + + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="not_ok", + message=f"Edit adds references with new domain(s): {domain_list}", + decision=AutoreviewDecision( + status="manual", + label="Requires manual review", + reason=f"Reference-only edit contains {domain_count} previously unused domain(s).", + ), + should_stop=True, + ) + + logger.info( + "Auto-approving reference-only edit %s with %s known domain(s)", + context.revision.revid, + len(checked_domains), + ) + ref_count = len(new_or_modified_refs) + domain_count = len(checked_domains) + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="ok", + message=f"Edit only modifies references ({ref_count} reference(s) with " + f"{domain_count} known domain(s)).", + decision=AutoreviewDecision( + status="approve", + label="Can be auto-approved", + reason="Edit only adds or modifies references with known domains.", + ), + should_stop=True, + ) diff --git a/app/reviews/autoreview/utils/wikitext.py b/app/reviews/autoreview/utils/wikitext.py index e65ce87a..ad8ef441 100644 --- a/app/reviews/autoreview/utils/wikitext.py +++ b/app/reviews/autoreview/utils/wikitext.py @@ -52,7 +52,7 @@ def extract_additions(parent_wikitext: str, pending_wikitext: str) -> list[str]: def get_parent_wikitext(revision: PendingRevision) -> str: - """Get parent revision wikitext from local database.""" + """Get parent revision wikitext from local database or API.""" cached_parent = getattr(revision, "parent_wikitext", None) if isinstance(cached_parent, str) and cached_parent: return cached_parent @@ -67,9 +67,143 @@ def get_parent_wikitext(revision: PendingRevision) -> str: parent_revision = PR.objects.get(page=revision.page, revid=parentid) return parent_revision.get_wikitext() except Exception: - logger.warning( - "Parent revision %s not found in local database for revision %s", + logger.info( + "Parent revision %s not in local database, fetching from API for revision %s", revision.parentid, revision.revid, ) + try: + import pywikibot + + wiki = revision.page.wiki + site = pywikibot.Site(code=wiki.code, fam=wiki.family) + request = site.simple_request( + action="query", + prop="revisions", + revids=str(parentid), + rvprop="content", + rvslots="main", + formatversion=2, + ) + response = request.submit() + pages = response.get("query", {}).get("pages", []) + + if pages and len(pages) > 0: + revisions = pages[0].get("revisions", []) + if revisions and len(revisions) > 0: + slots = revisions[0].get("slots", {}) + main_slot = slots.get("main", {}) + content = main_slot.get("content", "") + if content: + logger.info("Fetched parent revision %s from API", parentid) + return content + + logger.warning("Could not fetch parent revision %s from API", parentid) + return "" + except Exception as e: + logger.exception("Error fetching parent revision %s from API: %s", parentid, e) + return "" + + +def extract_references(text: str) -> list[str]: + """Extract all reference tags from wikitext.""" + if not text: + return [] + + references = [] + + # Extract self-closing refs + for match in re.finditer(r"]*/>", text, re.IGNORECASE): + references.append(match.group(0)) + + # Extract paired refs (excluding self-closing) + for match in re.finditer(r")[^>])*>(?:.*?)", text, re.IGNORECASE | re.DOTALL): + references.append(match.group(0)) + + return references + + +def strip_references(text: str) -> str: + """Remove all reference tags from wikitext.""" + if not text: return "" + + # First remove self-closing ref tags: + text = re.sub(r"]*/>", "", text, flags=re.IGNORECASE) + + # Then remove paired ref tags: content + # The opening tag must NOT end with />, so we use a negative lookahead + text = re.sub(r")[^>])*>(?:.*?)", "", text, flags=re.IGNORECASE | re.DOTALL) + + return text + + +def is_reference_only_edit( + parent_wikitext: str, pending_wikitext: str +) -> tuple[bool, bool, list[str]]: + """Check if edit only modifies references without changing other content. + + Returns: + tuple: (is_reference_only, has_removals, added_or_modified_refs) + - is_reference_only: True if only references changed + - has_removals: True if any references were removed + - added_or_modified_refs: List of new/modified reference content + """ + if not pending_wikitext: + return False, False, [] + + parent_without_refs = strip_references(parent_wikitext or "") + pending_without_refs = strip_references(pending_wikitext) + + # Don't normalize whitespace too aggressively - preserve structure + # Only collapse consecutive spaces/tabs on the same line + parent_normalized = re.sub(r"[ \t]+", " ", parent_without_refs).strip() + pending_normalized = re.sub(r"[ \t]+", " ", pending_without_refs).strip() + + if parent_normalized != pending_normalized: + return False, False, [] + + parent_refs = set(extract_references(parent_wikitext or "")) + pending_refs = set(extract_references(pending_wikitext)) + + if not parent_refs and not pending_refs: + return False, False, [] + + has_removals = len(parent_refs - pending_refs) > 0 + added_or_modified = list(pending_refs - parent_refs) + + if not added_or_modified and not has_removals: + return False, False, [] + + return True, has_removals, added_or_modified + + +def extract_urls_from_references(references: list[str]) -> list[str]: + """Extract all URLs from reference tags.""" + urls = [] + url_pattern = r'https?://[^\s\]<>"\'\|\{\}]+(?:\([^\s\)]*\))?' + + for ref in references: + for match in re.finditer(url_pattern, ref, re.IGNORECASE): + url = match.group(0) + url = url.rstrip(".,;:!?}") + urls.append(url) + + return urls + + +def extract_domain_from_url(url: str) -> str | None: + """Extract domain from URL without protocol, path, or query string.""" + from urllib.parse import urlparse + + try: + parsed = urlparse(url) + domain = parsed.netloc.lower() + + if domain.startswith("www."): + domain = domain[4:] + + return domain if domain else None + except Exception: + logger.warning("Failed to parse URL: %s", url) + return None diff --git a/app/reviews/services/wiki_client.py b/app/reviews/services/wiki_client.py index 150885ec..2b05bdb0 100644 --- a/app/reviews/services/wiki_client.py +++ b/app/reviews/services/wiki_client.py @@ -357,3 +357,28 @@ def fetch_review_statistics(self, days: int = 30) -> dict: """ stats_client = StatisticsClient(wiki=self.wiki, site=self.site) return stats_client.fetch_all_statistics(days=days, clear_existing=True) + + def has_domain_been_used(self, domain: str, exclude_page_id: int | None = None) -> bool: + """Check if domain has been used in Wikipedia articles (namespace=0). + + Args: + domain: The domain to check for usage + exclude_page_id: Optional page ID to exclude from the search + (to avoid matching the current page being checked) + """ + if not domain: + return False + + try: + ext_url_usage = self.site.exturlusage(url=domain, namespaces=[0], total=10) + + for page in ext_url_usage: + # Skip the page being checked to avoid self-matches + if exclude_page_id and hasattr(page, "pageid") and page.pageid == exclude_page_id: + continue + return True + + return False + except Exception: + logger.exception("Failed to check domain usage for: %s", domain) + return False diff --git a/app/reviews/tests/autoreview/test_reference_only_edit.py b/app/reviews/tests/autoreview/test_reference_only_edit.py new file mode 100644 index 00000000..acb7f562 --- /dev/null +++ b/app/reviews/tests/autoreview/test_reference_only_edit.py @@ -0,0 +1,482 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +from django.test import TestCase + +from reviews.autoreview.checks.reference_only_edit import check_reference_only_edit +from reviews.autoreview.context import CheckContext +from reviews.autoreview.utils.wikitext import ( + extract_domain_from_url, + extract_references, + extract_urls_from_references, + is_reference_only_edit, + strip_references, +) +from reviews.models import PendingPage, PendingRevision, Wiki, WikiConfiguration + + +class WikitextUtilityTests(TestCase): + def test_extract_references_basic(self): + text = "Some text Citation 1 more text Citation 2" + refs = extract_references(text) + self.assertEqual(len(refs), 2) + self.assertIn("Citation 1", refs) + self.assertIn("Citation 2", refs) + + def test_extract_references_with_attributes(self): + text = 'Citation Note' + refs = extract_references(text) + self.assertEqual(len(refs), 2) + self.assertIn('Citation', refs) + self.assertIn('Note', refs) + + def test_extract_references_self_closing(self): + text = 'Text more text ' + refs = extract_references(text) + self.assertEqual(len(refs), 2) + + def test_extract_references_multiline(self): + text = """Text + Long citation + with multiple lines + more text""" + refs = extract_references(text) + self.assertEqual(len(refs), 1) + + def test_strip_references(self): + text = "Text Citation more text" + result = strip_references(text) + self.assertEqual(result, "Text more text") + + def test_strip_references_self_closing(self): + text = 'Text more text' + result = strip_references(text) + self.assertNotIn("http://example.com/page", + "Text https://another.org text", + ] + urls = extract_urls_from_references(refs) + self.assertEqual(len(urls), 2) + self.assertIn("http://example.com/page", urls) + self.assertIn("https://another.org", urls) + + def test_extract_domain_from_url(self): + self.assertEqual(extract_domain_from_url("http://example.com/page"), "example.com") + self.assertEqual(extract_domain_from_url("https://www.test.org/path"), "test.org") + self.assertEqual( + extract_domain_from_url("https://subdomain.example.com"), "subdomain.example.com" + ) + + def test_is_reference_only_edit_adding_reference(self): + parent = "Some article text here." + pending = "Some article text here.New citation" + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertFalse(has_removals) + self.assertEqual(len(added), 1) + + def test_is_reference_only_edit_modifying_reference(self): + parent = "Text Old citation more text" + pending = "Text New citation more text" + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertTrue(has_removals) + self.assertEqual(len(added), 1) + + def test_is_reference_only_edit_changing_content(self): + parent = "Original text Citation" + pending = "Modified text Citation" + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertFalse(is_ref_only) + + def test_is_reference_only_edit_removing_reference(self): + parent = "Text Citation more text" + pending = "Text more text" + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertTrue(has_removals) + self.assertEqual(len(added), 0) + + def test_is_reference_only_edit_replacing_reference(self): + parent = "Text Old citation more text" + pending = "Text New citation more text" + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertTrue(has_removals) + self.assertEqual(len(added), 1) + + +class ReferenceOnlyEditCheckTests(TestCase): + def setUp(self): + self.wiki = Wiki.objects.create( + name="Test Wiki", + code="test", + family="wikipedia", + api_endpoint="https://test.wikipedia.org/w/api.php", + ) + WikiConfiguration.objects.create(wiki=self.wiki) + + self.page = PendingPage.objects.create( + wiki=self.wiki, + pageid=1, + title="Test Page", + stable_revid=100, + ) + + def _create_revision(self, revid, parentid, wikitext, parent_wikitext=None): + revision = PendingRevision.objects.create( + page=self.page, + revid=revid, + parentid=parentid, + user_name="TestEditor", + user_id=1, + timestamp=datetime.now(timezone.utc), + fetched_at=datetime.now(timezone.utc), + age_at_fetch=timedelta(hours=1), + sha1="test", + comment="Test edit", + change_tags=[], + wikitext=wikitext, + categories=[], + ) + + # Store parent wikitext for testing + if parent_wikitext is not None: + revision.parent_wikitext = parent_wikitext + + return revision + + def test_adding_single_reference_without_url(self): + parent_wikitext = "Article content here." + pending_wikitext = "Article content here.Smith, John (2020). Book Title." + + revision = self._create_revision(101, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + self.assertTrue(result.should_stop) + self.assertIn("reference", result.message.lower()) + + def test_adding_reference_with_known_domain(self): + parent_wikitext = "Article content." + pending_wikitext = "Article content.http://example.com/citation" + + revision = self._create_revision(102, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + mock_client.has_domain_been_used.return_value = True + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + self.assertTrue(result.should_stop) + mock_client.has_domain_been_used.assert_called_once_with("example.com", exclude_page_id=1) + + def test_adding_reference_with_new_domain(self): + parent_wikitext = "Article content." + pending_wikitext = "Article content.http://newdomain.com/source" + + revision = self._create_revision(103, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + mock_client.has_domain_been_used.return_value = False + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertTrue(result.should_stop) + self.assertIn("new domain", result.message.lower()) + + def test_modifying_existing_reference(self): + parent_wikitext = "Text Old citation more text" + pending_wikitext = "Text Updated citation more text" + + revision = self._create_revision(104, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_adding_multiple_references(self): + parent_wikitext = "Article text. More text." + pending_wikitext = "Article text.Citation 1 More text.Citation 2" + + revision = self._create_revision(105, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + self.assertTrue(result.should_stop) + + def test_removing_reference_only(self): + parent_wikitext = "Text Citation more text" + pending_wikitext = "Text more text" + + revision = self._create_revision(106, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertIn("only removes references", result.message.lower()) + + def test_mixed_content_and_reference_changes(self): + parent_wikitext = "Original content Citation" + pending_wikitext = "Modified content New citation" + + revision = self._create_revision(107, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "skip") + self.assertIn("beyond references", result.message.lower()) + + def test_self_closing_reference_tags(self): + parent_wikitext = "Article text." + pending_wikitext = 'Article text.' + + revision = self._create_revision(108, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_references_with_name_attribute(self): + parent_wikitext = "Article text." + pending_wikitext = 'Article text.Smith (2020)' + + revision = self._create_revision(109, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_references_with_group_attribute(self): + parent_wikitext = "Article text." + pending_wikitext = 'Article text.Footnote text' + + revision = self._create_revision(110, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_multiple_domains_mixed_known_and_new(self): + parent_wikitext = "Article text. More text." + pending_wikitext = ( + "Article text.http://known.com/page " + "More text.http://newdomain.org/page" + ) + + revision = self._create_revision(111, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + # First domain is known, second is new + mock_client.has_domain_been_used.side_effect = [True, False] + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertIn("new domain", result.message.lower()) + + def test_no_parent_revision(self): + pending_wikitext = "New article.Citation" + + revision = self._create_revision(112, None, pending_wikitext, "") + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + # Should handle gracefully - may skip or process as new content + self.assertIn(result.status, ["skip", "ok"]) + + def test_replacing_reference_with_different_one(self): + parent_wikitext = "Text Old source more text" + pending_wikitext = "Text New source more text" + + revision = self._create_revision(113, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_partial_reference_removal(self): + parent_wikitext = "Text.Ref1 More.Ref2 End.Ref3" + pending_wikitext = "Text.Ref1 More. End.Ref3" + + revision = self._create_revision(114, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertIn("only removes references", result.message.lower()) + + def test_adding_and_modifying_references_together(self): + parent_wikitext = "Text. More." + pending_wikitext = "Text.New ref 1 More.New ref 2" + + revision = self._create_revision(115, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve")