From dc1b418d741355c822077ff30224ec9a3e5320e4 Mon Sep 17 00:00:00 2001 From: Harshita Date: Mon, 27 Oct 2025 21:58:26 +0530 Subject: [PATCH 1/7] added ref only autoreview --- app/reviews/autoreview/checks/__init__.py | 7 + .../autoreview/checks/reference_only_edit.py | 127 ++++++ app/reviews/autoreview/utils/wikitext.py | 80 ++++ app/reviews/services/wiki_client.py | 23 + .../autoreview/test_reference_only_edit.py | 425 ++++++++++++++++++ 5 files changed, 662 insertions(+) create mode 100644 app/reviews/autoreview/checks/reference_only_edit.py create mode 100644 app/reviews/tests/autoreview/test_reference_only_edit.py diff --git a/app/reviews/autoreview/checks/__init__.py b/app/reviews/autoreview/checks/__init__.py index d75c2513..d0859d90 100644 --- a/app/reviews/autoreview/checks/__init__.py +++ b/app/reviews/autoreview/checks/__init__.py @@ -7,6 +7,7 @@ from .invalid_isbn import check_invalid_isbn from .manual_unapproval import check_manual_unapproval from .ores_scores import check_ores_scores +from .reference_only_edit import check_reference_only_edit from .render_errors import check_render_errors from .superseded_additions import check_superseded_additions from .user_block import check_user_block @@ -60,6 +61,12 @@ "function": check_invalid_isbn, "priority": 8, }, + { + "id": "reference-only-edit", + "name": "Reference-only edit", + "function": check_reference_only_edit, + "priority": 8.5, + }, { "id": "superseded-additions", "name": "Superseded additions", diff --git a/app/reviews/autoreview/checks/reference_only_edit.py b/app/reviews/autoreview/checks/reference_only_edit.py new file mode 100644 index 00000000..a5c31710 --- /dev/null +++ b/app/reviews/autoreview/checks/reference_only_edit.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import logging + +from ..base import CheckResult +from ..context import CheckContext +from ..decision import AutoreviewDecision +from ..utils.wikitext import ( + extract_domain_from_url, + extract_references, + extract_urls_from_references, + get_parent_wikitext, + is_reference_only_edit, +) + +logger = logging.getLogger(__name__) + + +def check_reference_only_edit(context: CheckContext) -> CheckResult: + """Check if revision only adds or modifies references.""" + pending_wikitext = context.revision.get_wikitext() + parent_wikitext = get_parent_wikitext(context.revision) + + if not is_reference_only_edit(parent_wikitext, pending_wikitext): + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="skip", + message="Edit modifies content beyond references.", + ) + + parent_refs = set(extract_references(parent_wikitext or "")) + pending_refs = set(extract_references(pending_wikitext)) + new_or_modified_refs = [ref for ref in pending_refs if ref not in parent_refs] + + if not new_or_modified_refs: + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="skip", + message="No new or modified references detected.", + ) + + urls = extract_urls_from_references(new_or_modified_refs) + + if not urls: + logger.info( + "Auto-approving reference-only edit %s (no URLs in new references)", + context.revision.revid, + ) + ref_count = len(new_or_modified_refs) + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="ok", + message=f"Edit only modifies references ({ref_count} reference(s) added/modified).", + decision=AutoreviewDecision( + status="approve", + label="Can be auto-approved", + reason="Edit only adds or modifies references without external URLs.", + ), + should_stop=True, + ) + + domains = [] + for url in urls: + domain = extract_domain_from_url(url) + if domain: + domains.append(domain) + + new_domains = [] + checked_domains = set() + + for domain in domains: + if domain in checked_domains: + continue + checked_domains.add(domain) + + has_been_used = context.client.has_domain_been_used(domain) + + if not has_been_used: + new_domains.append(domain) + logger.info( + "Domain %s has not been used before in revision %s", + domain, + context.revision.revid, + ) + + if new_domains: + domain_list = ", ".join(new_domains[:3]) + if len(new_domains) > 3: + domain_list += "..." + domain_count = len(new_domains) + + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="not_ok", + message=f"Edit adds references with new domain(s): {domain_list}", + decision=AutoreviewDecision( + status="manual", + label="Requires manual review", + reason=f"Reference-only edit contains {domain_count} previously unused domain(s).", + ), + should_stop=True, + ) + + logger.info( + "Auto-approving reference-only edit %s with %s known domain(s)", + context.revision.revid, + len(checked_domains), + ) + ref_count = len(new_or_modified_refs) + domain_count = len(checked_domains) + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="ok", + message=f"Edit only modifies references ({ref_count} reference(s) with " + f"{domain_count} known domain(s)).", + decision=AutoreviewDecision( + status="approve", + label="Can be auto-approved", + reason="Edit only adds or modifies references with known domains.", + ), + should_stop=True, + ) diff --git a/app/reviews/autoreview/utils/wikitext.py b/app/reviews/autoreview/utils/wikitext.py index e65ce87a..5fb52c79 100644 --- a/app/reviews/autoreview/utils/wikitext.py +++ b/app/reviews/autoreview/utils/wikitext.py @@ -73,3 +73,83 @@ def get_parent_wikitext(revision: PendingRevision) -> str: revision.revid, ) return "" + + +def extract_references(text: str) -> list[str]: + """Extract all reference tags from wikitext.""" + if not text: + return [] + + references = [] + ref_pattern = r"]*>.*?" + references.extend(re.findall(ref_pattern, text, flags=re.DOTALL | re.IGNORECASE)) + self_closing_pattern = r"]*/>" + references.extend(re.findall(self_closing_pattern, text, flags=re.IGNORECASE)) + + return references + + +def strip_references(text: str) -> str: + """Remove all reference tags from wikitext.""" + if not text: + return "" + + text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE) + text = re.sub(r"]*/>\s*", "", text, flags=re.IGNORECASE) + + return text + + +def is_reference_only_edit(parent_wikitext: str, pending_wikitext: str) -> bool: + """Check if edit only modifies references without changing other content.""" + if not pending_wikitext: + return False + + parent_without_refs = strip_references(parent_wikitext or "") + pending_without_refs = strip_references(pending_wikitext) + + parent_normalized = re.sub(r"\s+", " ", parent_without_refs).strip() + pending_normalized = re.sub(r"\s+", " ", pending_without_refs).strip() + + if parent_normalized != pending_normalized: + return False + + parent_refs = extract_references(parent_wikitext or "") + pending_refs = extract_references(pending_wikitext) + + if parent_refs and not pending_refs: + return False + + if not parent_refs and not pending_refs: + return False + + return True + + +def extract_urls_from_references(references: list[str]) -> list[str]: + """Extract all URLs from reference tags.""" + urls = [] + url_pattern = r"https?://[^\s<>\"\'\]\|]+" + + for ref in references: + found_urls = re.findall(url_pattern, ref, flags=re.IGNORECASE) + urls.extend(found_urls) + + return urls + + +def extract_domain_from_url(url: str) -> str | None: + """Extract domain from URL without protocol, path, or query string.""" + from urllib.parse import urlparse + + try: + parsed = urlparse(url) + domain = parsed.netloc.lower() + + if domain.startswith("www."): + domain = domain[4:] + + return domain if domain else None + except Exception: + logger.warning("Failed to parse URL: %s", url) + return None diff --git a/app/reviews/services/wiki_client.py b/app/reviews/services/wiki_client.py index fe0e07d1..6978a8ed 100644 --- a/app/reviews/services/wiki_client.py +++ b/app/reviews/services/wiki_client.py @@ -357,3 +357,26 @@ def fetch_review_statistics(self, days: int = 30) -> dict: """ stats_client = StatisticsClient(wiki=self.wiki, site=self.site) return stats_client.fetch_all_statistics(days=days, clear_existing=True) + + def has_domain_been_used(self, domain: str) -> bool: + """Check if domain has been used in Wikipedia articles (namespace=0).""" + if not domain: + return False + + try: + request = self.site.simple_request( + action="query", + list="exturlusage", + euprotocol="http", + euquery=domain, + eunamespace=0, + eulimit=1, + formatversion=2, + ) + response = request.submit() + pages = response.get("query", {}).get("exturlusage", []) + + return len(pages) > 0 + except Exception: + logger.exception("Failed to check domain usage for: %s", domain) + return False diff --git a/app/reviews/tests/autoreview/test_reference_only_edit.py b/app/reviews/tests/autoreview/test_reference_only_edit.py new file mode 100644 index 00000000..2decde67 --- /dev/null +++ b/app/reviews/tests/autoreview/test_reference_only_edit.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +from django.test import TestCase + +from reviews.autoreview.checks.reference_only_edit import check_reference_only_edit +from reviews.autoreview.context import CheckContext +from reviews.autoreview.utils.wikitext import ( + extract_domain_from_url, + extract_references, + extract_urls_from_references, + is_reference_only_edit, + strip_references, +) +from reviews.models import PendingPage, PendingRevision, Wiki, WikiConfiguration + + +class WikitextUtilityTests(TestCase): + def test_extract_references_basic(self): + text = "Some text Citation 1 more text Citation 2" + refs = extract_references(text) + self.assertEqual(len(refs), 2) + self.assertIn("Citation 1", refs) + self.assertIn("Citation 2", refs) + + def test_extract_references_with_attributes(self): + text = 'Citation Note' + refs = extract_references(text) + self.assertEqual(len(refs), 2) + self.assertIn('Citation', refs) + self.assertIn('Note', refs) + + def test_extract_references_self_closing(self): + text = 'Text more text ' + refs = extract_references(text) + self.assertEqual(len(refs), 2) + + def test_extract_references_multiline(self): + text = """Text + Long citation + with multiple lines + more text""" + refs = extract_references(text) + self.assertEqual(len(refs), 1) + + def test_strip_references(self): + text = "Text Citation more text" + result = strip_references(text) + self.assertEqual(result, "Text more text") + + def test_strip_references_self_closing(self): + text = 'Text more text' + result = strip_references(text) + self.assertNotIn("http://example.com/page", + "Text https://another.org text", + ] + urls = extract_urls_from_references(refs) + self.assertEqual(len(urls), 2) + self.assertIn("http://example.com/page", urls) + self.assertIn("https://another.org", urls) + + def test_extract_domain_from_url(self): + self.assertEqual(extract_domain_from_url("http://example.com/page"), "example.com") + self.assertEqual(extract_domain_from_url("https://www.test.org/path"), "test.org") + self.assertEqual( + extract_domain_from_url("https://subdomain.example.com"), "subdomain.example.com" + ) + + def test_is_reference_only_edit_adding_reference(self): + parent = "Some article text here." + pending = "Some article text here.New citation" + self.assertTrue(is_reference_only_edit(parent, pending)) + + def test_is_reference_only_edit_modifying_reference(self): + parent = "Text Old citation more text" + pending = "Text New citation more text" + self.assertTrue(is_reference_only_edit(parent, pending)) + + def test_is_reference_only_edit_changing_content(self): + parent = "Original text Citation" + pending = "Modified text Citation" + self.assertFalse(is_reference_only_edit(parent, pending)) + + def test_is_reference_only_edit_removing_reference(self): + parent = "Text Citation more text" + pending = "Text more text" + self.assertFalse(is_reference_only_edit(parent, pending)) + + def test_is_reference_only_edit_replacing_reference(self): + parent = "Text Old citation more text" + pending = "Text New citation more text" + self.assertTrue(is_reference_only_edit(parent, pending)) + + +class ReferenceOnlyEditCheckTests(TestCase): + def setUp(self): + self.wiki = Wiki.objects.create( + name="Test Wiki", + code="test", + family="wikipedia", + api_endpoint="https://test.wikipedia.org/w/api.php", + ) + WikiConfiguration.objects.create(wiki=self.wiki) + + self.page = PendingPage.objects.create( + wiki=self.wiki, + pageid=1, + title="Test Page", + stable_revid=100, + ) + + def _create_revision(self, revid, parentid, wikitext, parent_wikitext=None): + revision = PendingRevision.objects.create( + page=self.page, + revid=revid, + parentid=parentid, + user_name="TestEditor", + user_id=1, + timestamp=datetime.now(timezone.utc), + fetched_at=datetime.now(timezone.utc), + age_at_fetch=timedelta(hours=1), + sha1="test", + comment="Test edit", + change_tags=[], + wikitext=wikitext, + categories=[], + ) + + # Store parent wikitext for testing + if parent_wikitext is not None: + revision.parent_wikitext = parent_wikitext + + return revision + + def test_adding_single_reference_without_url(self): + parent_wikitext = "Article content here." + pending_wikitext = "Article content here.Smith, John (2020). Book Title." + + revision = self._create_revision(101, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + self.assertTrue(result.should_stop) + self.assertIn("reference", result.message.lower()) + + def test_adding_reference_with_known_domain(self): + parent_wikitext = "Article content." + pending_wikitext = "Article content.http://example.com/citation" + + revision = self._create_revision(102, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + mock_client.has_domain_been_used.return_value = True + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + self.assertTrue(result.should_stop) + mock_client.has_domain_been_used.assert_called_once_with("example.com") + + def test_adding_reference_with_new_domain(self): + parent_wikitext = "Article content." + pending_wikitext = "Article content.http://newdomain.com/source" + + revision = self._create_revision(103, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + mock_client.has_domain_been_used.return_value = False + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertTrue(result.should_stop) + self.assertIn("new domain", result.message.lower()) + + def test_modifying_existing_reference(self): + parent_wikitext = "Text Old citation more text" + pending_wikitext = "Text Updated citation more text" + + revision = self._create_revision(104, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_adding_multiple_references(self): + parent_wikitext = "Article text. More text." + pending_wikitext = "Article text.Citation 1 More text.Citation 2" + + revision = self._create_revision(105, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + self.assertTrue(result.should_stop) + + def test_removing_reference_only(self): + parent_wikitext = "Text Citation more text" + pending_wikitext = "Text more text" + + revision = self._create_revision(106, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "skip") + self.assertIn("beyond references", result.message.lower()) + + def test_mixed_content_and_reference_changes(self): + parent_wikitext = "Original content Citation" + pending_wikitext = "Modified content New citation" + + revision = self._create_revision(107, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "skip") + self.assertIn("beyond references", result.message.lower()) + + def test_self_closing_reference_tags(self): + parent_wikitext = "Article text." + pending_wikitext = 'Article text.' + + revision = self._create_revision(108, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_references_with_name_attribute(self): + parent_wikitext = "Article text." + pending_wikitext = 'Article text.Smith (2020)' + + revision = self._create_revision(109, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_references_with_group_attribute(self): + parent_wikitext = "Article text." + pending_wikitext = 'Article text.Footnote text' + + revision = self._create_revision(110, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") + + def test_multiple_domains_mixed_known_and_new(self): + parent_wikitext = "Article text. More text." + pending_wikitext = ( + "Article text.http://known.com/page " + "More text.http://newdomain.org/page" + ) + + revision = self._create_revision(111, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + # First domain is known, second is new + mock_client.has_domain_been_used.side_effect = [True, False] + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertIn("new domain", result.message.lower()) + + def test_no_parent_revision(self): + pending_wikitext = "New article.Citation" + + revision = self._create_revision(112, None, pending_wikitext, "") + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + # Should handle gracefully - may skip or process as new content + self.assertIn(result.status, ["skip", "ok"]) + + def test_replacing_reference_with_different_one(self): + parent_wikitext = "Text Old source more text" + pending_wikitext = "Text New source more text" + + revision = self._create_revision(113, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") From 671f86fdfefe36afb0a8be319f899187264222fe Mon Sep 17 00:00:00 2001 From: Harshita Date: Mon, 27 Oct 2025 23:41:34 +0530 Subject: [PATCH 2/7] fix bugs --- app/reviews/autoreview/utils/wikitext.py | 60 ++++++++++++++++++------ app/reviews/services/wiki_client.py | 17 +++---- app/user-config.py | 2 +- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/app/reviews/autoreview/utils/wikitext.py b/app/reviews/autoreview/utils/wikitext.py index 5fb52c79..d5d1b209 100644 --- a/app/reviews/autoreview/utils/wikitext.py +++ b/app/reviews/autoreview/utils/wikitext.py @@ -52,7 +52,7 @@ def extract_additions(parent_wikitext: str, pending_wikitext: str) -> list[str]: def get_parent_wikitext(revision: PendingRevision) -> str: - """Get parent revision wikitext from local database.""" + """Get parent revision wikitext from local database or API.""" cached_parent = getattr(revision, "parent_wikitext", None) if isinstance(cached_parent, str) and cached_parent: return cached_parent @@ -67,12 +67,42 @@ def get_parent_wikitext(revision: PendingRevision) -> str: parent_revision = PR.objects.get(page=revision.page, revid=parentid) return parent_revision.get_wikitext() except Exception: - logger.warning( - "Parent revision %s not found in local database for revision %s", + logger.info( + "Parent revision %s not in local database, fetching from API for revision %s", revision.parentid, revision.revid, ) - return "" + try: + import pywikibot + + wiki = revision.page.wiki + site = pywikibot.Site(code=wiki.code, fam=wiki.family) + request = site.simple_request( + action="query", + prop="revisions", + revids=str(parentid), + rvprop="content", + rvslots="main", + formatversion=2, + ) + response = request.submit() + pages = response.get("query", {}).get("pages", []) + + if pages and len(pages) > 0: + revisions = pages[0].get("revisions", []) + if revisions and len(revisions) > 0: + slots = revisions[0].get("slots", {}) + main_slot = slots.get("main", {}) + content = main_slot.get("content", "") + if content: + logger.info("Fetched parent revision %s from API", parentid) + return content + + logger.warning("Could not fetch parent revision %s from API", parentid) + return "" + except Exception as e: + logger.exception("Error fetching parent revision %s from API: %s", parentid, e) + return "" def extract_references(text: str) -> list[str]: @@ -81,10 +111,10 @@ def extract_references(text: str) -> list[str]: return [] references = [] - ref_pattern = r"]*>.*?" - references.extend(re.findall(ref_pattern, text, flags=re.DOTALL | re.IGNORECASE)) - self_closing_pattern = r"]*/>" - references.extend(re.findall(self_closing_pattern, text, flags=re.IGNORECASE)) + ref_pattern = r"]*)?>(?:.*?)|]*)?/>" + + for match in re.finditer(ref_pattern, text, re.IGNORECASE | re.DOTALL): + references.append(match.group(0)) return references @@ -94,10 +124,10 @@ def strip_references(text: str) -> str: if not text: return "" - text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"]*/>\s*", "", text, flags=re.IGNORECASE) + ref_pattern = r"]*)?>(?:.*?)|]*)?/>" + cleaned = re.sub(ref_pattern, "", text, flags=re.IGNORECASE | re.DOTALL) - return text + return cleaned def is_reference_only_edit(parent_wikitext: str, pending_wikitext: str) -> bool: @@ -129,11 +159,13 @@ def is_reference_only_edit(parent_wikitext: str, pending_wikitext: str) -> bool: def extract_urls_from_references(references: list[str]) -> list[str]: """Extract all URLs from reference tags.""" urls = [] - url_pattern = r"https?://[^\s<>\"\'\]\|]+" + url_pattern = r'https?://[^\s\]<>"\'\|\{\}]+(?:\([^\s\)]*\))?' for ref in references: - found_urls = re.findall(url_pattern, ref, flags=re.IGNORECASE) - urls.extend(found_urls) + for match in re.finditer(url_pattern, ref, re.IGNORECASE): + url = match.group(0) + url = url.rstrip(".,;:!?}") + urls.append(url) return urls diff --git a/app/reviews/services/wiki_client.py b/app/reviews/services/wiki_client.py index 6978a8ed..4a690834 100644 --- a/app/reviews/services/wiki_client.py +++ b/app/reviews/services/wiki_client.py @@ -364,19 +364,14 @@ def has_domain_been_used(self, domain: str) -> bool: return False try: - request = self.site.simple_request( - action="query", - list="exturlusage", - euprotocol="http", - euquery=domain, - eunamespace=0, - eulimit=1, - formatversion=2, + ext_url_usage = self.site.exturlusage( + url=domain, protocol="http", namespaces=[0], total=1 ) - response = request.submit() - pages = response.get("query", {}).get("exturlusage", []) - return len(pages) > 0 + for _ in ext_url_usage: + return True + + return False except Exception: logger.exception("Failed to check domain usage for: %s", domain) return False diff --git a/app/user-config.py b/app/user-config.py index a2cb3570..dd360c97 100644 --- a/app/user-config.py +++ b/app/user-config.py @@ -1 +1 @@ -usernames["meta"]["meta"] = "WIKIMEDIA_USERNAME" +usernames["meta"]["meta"] = "Harshita 2208" From 74cc1920a2531f0ec00766832bd3edaefac8d9f5 Mon Sep 17 00:00:00 2001 From: Harshita Date: Mon, 27 Oct 2025 23:41:56 +0530 Subject: [PATCH 3/7] revert changes --- app/user-config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/user-config.py b/app/user-config.py index dd360c97..a2cb3570 100644 --- a/app/user-config.py +++ b/app/user-config.py @@ -1 +1 @@ -usernames["meta"]["meta"] = "Harshita 2208" +usernames["meta"]["meta"] = "WIKIMEDIA_USERNAME" From b4a7512f42940bd8ad9e5b2988fd78d77ef319c4 Mon Sep 17 00:00:00 2001 From: Harshita Date: Mon, 27 Oct 2025 23:53:16 +0530 Subject: [PATCH 4/7] fixing some test cases --- .../autoreview/checks/reference_only_edit.py | 23 ++++-- app/reviews/autoreview/utils/wikitext.py | 34 ++++++--- .../autoreview/test_reference_only_edit.py | 71 +++++++++++++++++-- 3 files changed, 105 insertions(+), 23 deletions(-) diff --git a/app/reviews/autoreview/checks/reference_only_edit.py b/app/reviews/autoreview/checks/reference_only_edit.py index a5c31710..1551521a 100644 --- a/app/reviews/autoreview/checks/reference_only_edit.py +++ b/app/reviews/autoreview/checks/reference_only_edit.py @@ -7,7 +7,6 @@ from ..decision import AutoreviewDecision from ..utils.wikitext import ( extract_domain_from_url, - extract_references, extract_urls_from_references, get_parent_wikitext, is_reference_only_edit, @@ -21,7 +20,11 @@ def check_reference_only_edit(context: CheckContext) -> CheckResult: pending_wikitext = context.revision.get_wikitext() parent_wikitext = get_parent_wikitext(context.revision) - if not is_reference_only_edit(parent_wikitext, pending_wikitext): + is_ref_only, has_removals, new_or_modified_refs = is_reference_only_edit( + parent_wikitext, pending_wikitext + ) + + if not is_ref_only: return CheckResult( check_id="reference-only-edit", check_title="Reference-only edit detection", @@ -29,9 +32,19 @@ def check_reference_only_edit(context: CheckContext) -> CheckResult: message="Edit modifies content beyond references.", ) - parent_refs = set(extract_references(parent_wikitext or "")) - pending_refs = set(extract_references(pending_wikitext)) - new_or_modified_refs = [ref for ref in pending_refs if ref not in parent_refs] + if has_removals and not new_or_modified_refs: + return CheckResult( + check_id="reference-only-edit", + check_title="Reference-only edit detection", + status="not_ok", + message="Edit only removes references without adding new ones.", + decision=AutoreviewDecision( + status="manual", + label="Requires manual review", + reason="Reference-only edits that only remove references require manual review.", + ), + should_stop=True, + ) if not new_or_modified_refs: return CheckResult( diff --git a/app/reviews/autoreview/utils/wikitext.py b/app/reviews/autoreview/utils/wikitext.py index d5d1b209..6aebaa4e 100644 --- a/app/reviews/autoreview/utils/wikitext.py +++ b/app/reviews/autoreview/utils/wikitext.py @@ -130,10 +130,19 @@ def strip_references(text: str) -> str: return cleaned -def is_reference_only_edit(parent_wikitext: str, pending_wikitext: str) -> bool: - """Check if edit only modifies references without changing other content.""" +def is_reference_only_edit( + parent_wikitext: str, pending_wikitext: str +) -> tuple[bool, bool, list[str]]: + """Check if edit only modifies references without changing other content. + + Returns: + tuple: (is_reference_only, has_removals, added_or_modified_refs) + - is_reference_only: True if only references changed + - has_removals: True if any references were removed + - added_or_modified_refs: List of new/modified reference content + """ if not pending_wikitext: - return False + return False, False, [] parent_without_refs = strip_references(parent_wikitext or "") pending_without_refs = strip_references(pending_wikitext) @@ -142,18 +151,21 @@ def is_reference_only_edit(parent_wikitext: str, pending_wikitext: str) -> bool: pending_normalized = re.sub(r"\s+", " ", pending_without_refs).strip() if parent_normalized != pending_normalized: - return False + return False, False, [] - parent_refs = extract_references(parent_wikitext or "") - pending_refs = extract_references(pending_wikitext) - - if parent_refs and not pending_refs: - return False + parent_refs = set(extract_references(parent_wikitext or "")) + pending_refs = set(extract_references(pending_wikitext)) if not parent_refs and not pending_refs: - return False + return False, False, [] + + has_removals = len(parent_refs - pending_refs) > 0 + added_or_modified = list(pending_refs - parent_refs) + + if not added_or_modified and not has_removals: + return False, False, [] - return True + return True, has_removals, added_or_modified def extract_urls_from_references(references: list[str]) -> list[str]: diff --git a/app/reviews/tests/autoreview/test_reference_only_edit.py b/app/reviews/tests/autoreview/test_reference_only_edit.py index 2decde67..4557c2e7 100644 --- a/app/reviews/tests/autoreview/test_reference_only_edit.py +++ b/app/reviews/tests/autoreview/test_reference_only_edit.py @@ -75,27 +75,40 @@ def test_extract_domain_from_url(self): def test_is_reference_only_edit_adding_reference(self): parent = "Some article text here." pending = "Some article text here.New citation" - self.assertTrue(is_reference_only_edit(parent, pending)) + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertFalse(has_removals) + self.assertEqual(len(added), 1) def test_is_reference_only_edit_modifying_reference(self): parent = "Text Old citation more text" pending = "Text New citation more text" - self.assertTrue(is_reference_only_edit(parent, pending)) + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertTrue(has_removals) + self.assertEqual(len(added), 1) def test_is_reference_only_edit_changing_content(self): parent = "Original text Citation" pending = "Modified text Citation" - self.assertFalse(is_reference_only_edit(parent, pending)) + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertFalse(is_ref_only) def test_is_reference_only_edit_removing_reference(self): parent = "Text Citation more text" pending = "Text more text" - self.assertFalse(is_reference_only_edit(parent, pending)) + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertTrue(has_removals) + self.assertEqual(len(added), 0) def test_is_reference_only_edit_replacing_reference(self): parent = "Text Old citation more text" pending = "Text New citation more text" - self.assertTrue(is_reference_only_edit(parent, pending)) + is_ref_only, has_removals, added = is_reference_only_edit(parent, pending) + self.assertTrue(is_ref_only) + self.assertTrue(has_removals) + self.assertEqual(len(added), 1) class ReferenceOnlyEditCheckTests(TestCase): @@ -269,8 +282,9 @@ def test_removing_reference_only(self): ) result = check_reference_only_edit(context) - self.assertEqual(result.status, "skip") - self.assertIn("beyond references", result.message.lower()) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertIn("only removes references", result.message.lower()) def test_mixed_content_and_reference_changes(self): parent_wikitext = "Original content Citation" @@ -423,3 +437,46 @@ def test_replacing_reference_with_different_one(self): result = check_reference_only_edit(context) self.assertEqual(result.status, "ok") self.assertEqual(result.decision.status, "approve") + + def test_partial_reference_removal(self): + parent_wikitext = "Text.Ref1 More.Ref2 End.Ref3" + pending_wikitext = "Text.Ref1 More. End.Ref3" + + revision = self._create_revision(114, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "not_ok") + self.assertEqual(result.decision.status, "manual") + self.assertIn("only removes references", result.message.lower()) + + def test_adding_and_modifying_references_together(self): + parent_wikitext = "Text. More." + pending_wikitext = "Text.New ref 1 More.New ref 2" + + revision = self._create_revision(115, 100, pending_wikitext, parent_wikitext) + + mock_client = MagicMock() + + context = CheckContext( + revision=revision, + client=mock_client, + profile=None, + auto_groups={}, + blocking_categories={}, + redirect_aliases=[], + ) + + result = check_reference_only_edit(context) + self.assertEqual(result.status, "ok") + self.assertEqual(result.decision.status, "approve") From 1e99c8079a42d0eefc85b1191dbb12ed855440ec Mon Sep 17 00:00:00 2001 From: Harshita Date: Sat, 1 Nov 2025 07:35:44 +0530 Subject: [PATCH 5/7] fix from comments --- app/reviews/services/wiki_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/reviews/services/wiki_client.py b/app/reviews/services/wiki_client.py index 4a690834..328f9f48 100644 --- a/app/reviews/services/wiki_client.py +++ b/app/reviews/services/wiki_client.py @@ -364,9 +364,7 @@ def has_domain_been_used(self, domain: str) -> bool: return False try: - ext_url_usage = self.site.exturlusage( - url=domain, protocol="http", namespaces=[0], total=1 - ) + ext_url_usage = self.site.exturlusage(url=domain, namespaces=[0], total=1) for _ in ext_url_usage: return True From 650b8bbf7f9359fb5619ffbdf8e6a860bf767349 Mon Sep 17 00:00:00 2001 From: Harshita Date: Mon, 3 Nov 2025 17:12:09 +0530 Subject: [PATCH 6/7] fix from comments --- .../autoreview/checks/reference_only_edit.py | 5 ++++- app/reviews/services/wiki_client.py | 17 +++++++++++++---- .../autoreview/test_reference_only_edit.py | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/app/reviews/autoreview/checks/reference_only_edit.py b/app/reviews/autoreview/checks/reference_only_edit.py index 1551521a..01eb3589 100644 --- a/app/reviews/autoreview/checks/reference_only_edit.py +++ b/app/reviews/autoreview/checks/reference_only_edit.py @@ -84,12 +84,15 @@ def check_reference_only_edit(context: CheckContext) -> CheckResult: new_domains = [] checked_domains = set() + # Get the page ID to exclude from domain usage search (avoid self-matches) + page_id = context.revision.page.pageid if hasattr(context.revision.page, "pageid") else None + for domain in domains: if domain in checked_domains: continue checked_domains.add(domain) - has_been_used = context.client.has_domain_been_used(domain) + has_been_used = context.client.has_domain_been_used(domain, exclude_page_id=page_id) if not has_been_used: new_domains.append(domain) diff --git a/app/reviews/services/wiki_client.py b/app/reviews/services/wiki_client.py index e88edd22..2b05bdb0 100644 --- a/app/reviews/services/wiki_client.py +++ b/app/reviews/services/wiki_client.py @@ -358,15 +358,24 @@ def fetch_review_statistics(self, days: int = 30) -> dict: stats_client = StatisticsClient(wiki=self.wiki, site=self.site) return stats_client.fetch_all_statistics(days=days, clear_existing=True) - def has_domain_been_used(self, domain: str) -> bool: - """Check if domain has been used in Wikipedia articles (namespace=0).""" + def has_domain_been_used(self, domain: str, exclude_page_id: int | None = None) -> bool: + """Check if domain has been used in Wikipedia articles (namespace=0). + + Args: + domain: The domain to check for usage + exclude_page_id: Optional page ID to exclude from the search + (to avoid matching the current page being checked) + """ if not domain: return False try: - ext_url_usage = self.site.exturlusage(url=domain, namespaces=[0], total=1) + ext_url_usage = self.site.exturlusage(url=domain, namespaces=[0], total=10) - for _ in ext_url_usage: + for page in ext_url_usage: + # Skip the page being checked to avoid self-matches + if exclude_page_id and hasattr(page, "pageid") and page.pageid == exclude_page_id: + continue return True return False diff --git a/app/reviews/tests/autoreview/test_reference_only_edit.py b/app/reviews/tests/autoreview/test_reference_only_edit.py index 4557c2e7..acb7f562 100644 --- a/app/reviews/tests/autoreview/test_reference_only_edit.py +++ b/app/reviews/tests/autoreview/test_reference_only_edit.py @@ -195,7 +195,7 @@ def test_adding_reference_with_known_domain(self): self.assertEqual(result.status, "ok") self.assertEqual(result.decision.status, "approve") self.assertTrue(result.should_stop) - mock_client.has_domain_been_used.assert_called_once_with("example.com") + mock_client.has_domain_been_used.assert_called_once_with("example.com", exclude_page_id=1) def test_adding_reference_with_new_domain(self): parent_wikitext = "Article content." From 5be03dba56a43640ff6890106bd569126a082fa9 Mon Sep 17 00:00:00 2001 From: Harshita Date: Fri, 7 Nov 2025 01:59:45 +0530 Subject: [PATCH 7/7] fix from reviews --- app/reviews/autoreview/utils/wikitext.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/app/reviews/autoreview/utils/wikitext.py b/app/reviews/autoreview/utils/wikitext.py index 6aebaa4e..ad8ef441 100644 --- a/app/reviews/autoreview/utils/wikitext.py +++ b/app/reviews/autoreview/utils/wikitext.py @@ -111,9 +111,13 @@ def extract_references(text: str) -> list[str]: return [] references = [] - ref_pattern = r"]*)?>(?:.*?)|]*)?/>" - for match in re.finditer(ref_pattern, text, re.IGNORECASE | re.DOTALL): + # Extract self-closing refs + for match in re.finditer(r"]*/>", text, re.IGNORECASE): + references.append(match.group(0)) + + # Extract paired refs (excluding self-closing) + for match in re.finditer(r")[^>])*>(?:.*?)", text, re.IGNORECASE | re.DOTALL): references.append(match.group(0)) return references @@ -124,10 +128,14 @@ def strip_references(text: str) -> str: if not text: return "" - ref_pattern = r"]*)?>(?:.*?)|]*)?/>" - cleaned = re.sub(ref_pattern, "", text, flags=re.IGNORECASE | re.DOTALL) + # First remove self-closing ref tags: + text = re.sub(r"]*/>", "", text, flags=re.IGNORECASE) + + # Then remove paired ref tags: content + # The opening tag must NOT end with />, so we use a negative lookahead + text = re.sub(r")[^>])*>(?:.*?)", "", text, flags=re.IGNORECASE | re.DOTALL) - return cleaned + return text def is_reference_only_edit( @@ -147,8 +155,10 @@ def is_reference_only_edit( parent_without_refs = strip_references(parent_wikitext or "") pending_without_refs = strip_references(pending_wikitext) - parent_normalized = re.sub(r"\s+", " ", parent_without_refs).strip() - pending_normalized = re.sub(r"\s+", " ", pending_without_refs).strip() + # Don't normalize whitespace too aggressively - preserve structure + # Only collapse consecutive spaces/tabs on the same line + parent_normalized = re.sub(r"[ \t]+", " ", parent_without_refs).strip() + pending_normalized = re.sub(r"[ \t]+", " ", pending_without_refs).strip() if parent_normalized != pending_normalized: return False, False, []