From f90726d3e8b6b28efc9df7742f7c4b0a048f2fc5 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Tue, 28 Jul 2026 09:52:14 +0200 Subject: [PATCH 01/16] Add `qlever check-sync-with-wikidata` New command that checks whether entities on the endpoint are exactly in sync with wikidata.org, by comparing each entity against the canonical data from `Special:EntityData` (which renders the live revision on demand; deliberately NOT against the WDQS SPARQL endpoint, which is a replica with its own lag and no longer contains the scholarly subgraph). See the command's help text and the comments for the details. --- .../commands/check_sync_with_wikidata.py | 567 ++++++++++++++++++ 1 file changed, 567 insertions(+) create mode 100644 src/qlever/commands/check_sync_with_wikidata.py diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py new file mode 100644 index 00000000..4d1276b2 --- /dev/null +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -0,0 +1,567 @@ +from __future__ import annotations + +import glob +import gzip +import random +import re +import shutil +import subprocess +import tempfile +import time +import urllib.parse +import urllib.request +from pathlib import Path + +from qlever.command import QleverCommand +from qlever.log import log + +# User agent for all requests to wikidata.org (required by the API etiquette). +USER_AGENT = "qlever-check-sync/0.1 (https://github.com/qlever-dev/qlever)" + +WD = "http://www.wikidata.org/entity/" +SCHEMA = "http://schema.org/" +WIKIBASE = "http://wikiba.se/ontology#" +STATEMENT = "http://www.wikidata.org/entity/statement/" + +# Entity-level counter triples that are contained in the full dump (and hence +# in the index), but NOT in the output of `Special:EntityData`. They are +# excluded from the comparison on both sides. +EXCLUDED_ENTITY_PREDICATES = { + f"{WIKIBASE}sitelinks", + f"{WIKIBASE}statements", + f"{WIKIBASE}identifiers", +} + +# `munge.sh` drops the `rdf:type wikibase:Reference` triples from the dump, +# but the update stream contains them, so an index that has received updates +# has them for some references and not for others. They carry no information +# (every `wdref:` node is a reference), so they are excluded on both sides. +# The same heterogeneity exists for the `wikibase:quantityNormalized` links. +RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +EXCLUDED_TYPE_OBJECTS = {f"{WIKIBASE}Reference"} +EXCLUDED_PREDICATES = {f"{WIKIBASE}quantityNormalized"} + +# Geographic coordinates are stored by QLever in a fixed-precision encoding +# (roughly 1e-6 degrees) and exported in a normalized form, so they are +# compared after rounding to 6 decimal places. +GEO_COMPONENT_PREDICATES = { + f"{WIKIBASE}geoLatitude", + f"{WIKIBASE}geoLongitude", + f"{WIKIBASE}geoPrecision", +} +WKT_DATATYPE = "http://www.opengis.net/ont/geosparql#wktLiteral" + + +class CheckSyncWithWikidataCommand(QleverCommand): + """ + Class for executing the `check-sync-with-wikidata` command. + """ + + def __init__(self): + pass + + def description(self) -> str: + return ( + "Check that entities on this endpoint are exactly in sync with" + " wikidata.org, by comparing against `Special:EntityData`" + ) + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return {"server": ["host_name", "port"]} + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "--sparql-endpoint", + help="URL of the QLever server, default is {host_name}:{port}", + ) + subparser.add_argument( + "--entities", + help="Comma-separated list of entity IDs to check" + " (default: a random sample, see `--num-entities`)", + ) + subparser.add_argument( + "--num-entities", + type=int, + default=10, + help="Number of randomly sampled entities to check (default: 10)", + ) + subparser.add_argument( + "--recent-fraction", + type=float, + default=0.5, + help="Fraction of the sample drawn from recently edited entities" + " (they exercise the update path); the rest is drawn uniformly" + " (default: 0.5)", + ) + subparser.add_argument( + "--seed", + type=int, + default=42, + help="Seed for the random sample (default: 42)", + ) + subparser.add_argument( + "--munge", + choices=["auto", "yes", "no"], + default="auto", + help="Munge the canonical data with the `munge.sh` from the" + " `service-*` directory before comparing (`auto`: munge if such" + " a directory exists, which is the right thing for an index" + " built from the munged dump)", + ) + subparser.add_argument( + "--keep-files", + action="store_true", + default=False, + help="Keep the downloaded and munged files for inspection", + ) + + # SPARQL helpers. + + def sparql(self, endpoint, query, accept): + data = urllib.parse.urlencode({"query": query}).encode() + request = urllib.request.Request( + endpoint, + data=data, + headers={ + "Accept": accept, + "Content-Type": "application/x-www-form-urlencoded", + }, + ) + with urllib.request.urlopen(request, timeout=60) as response: + return response.read().decode() + + def qlever_entity_graph(self, endpoint, entity_id): + """ + Fetch the full document of the given entity from the QLever endpoint: + the triples with the entity as subject, the statement nodes reachable + from the entity (plus their references and values), and the sitelink + article blocks. + """ + from rdflib import Graph + + e = f"<{WD}{entity_id}>" + # NOTE: Statement IRIs of old statements contain the entity ID in + # lowercase. + st = ( + f'FILTER(STRSTARTS(STR(?st), "{STATEMENT}{entity_id}-")' + f' || STRSTARTS(STR(?st), "{STATEMENT}{entity_id.lower()}-"))' + ) + queries = [ + f"CONSTRUCT {{ {e} ?p ?o }} WHERE {{ {e} ?p ?o }}", + f"CONSTRUCT {{ ?st ?p2 ?o2 }} WHERE" + f" {{ {e} ?p1 ?st . {st} ?st ?p2 ?o2 }}", + f"CONSTRUCT {{ ?x ?p3 ?o3 }} WHERE" + f" {{ {e} ?p1 ?st . {st} ?st ?p2 ?x ." + f' FILTER(STRSTARTS(STR(?x), "http://www.wikidata.org/reference/")' + f' || STRSTARTS(STR(?x), "http://www.wikidata.org/value/"))' + f" ?x ?p3 ?o3 }}", + f"CONSTRUCT {{ ?x2 ?p5 ?o5 }} WHERE" + f" {{ {e} ?p1 ?st . {st} ?st ?p2 ?x ." + f' FILTER(STRSTARTS(STR(?x), "http://www.wikidata.org/reference/"))' + f" ?x ?p3 ?x2 ." + f' FILTER(STRSTARTS(STR(?x2), "http://www.wikidata.org/value/"))' + f" ?x2 ?p5 ?o5 }}", + f"CONSTRUCT {{ ?a ?p4 ?o4 }} WHERE" + f" {{ ?a <{SCHEMA}about> {e} . ?a ?p4 ?o4 }}", + f"CONSTRUCT {{ ?w ?p6 ?o6 }} WHERE" + f" {{ ?a <{SCHEMA}about> {e} ." + f" ?a <{SCHEMA}isPartOf> ?w . ?w ?p6 ?o6 }}", + ] + graph = Graph() + for query in queries: + turtle = self.sparql(endpoint, query, "text/turtle") + graph.parse(data=turtle, format="turtle") + return graph + + # Canonical data from wikidata.org. + + def fetch_canonical(self, entity_id): + """ + Download the canonical TTL for the entity from `Special:EntityData`. + Returns `(ttl_bytes, redirected_to)`, where `redirected_to` is not + `None` if the entity is a redirect. + """ + url = ( + "https://www.wikidata.org/wiki/Special:EntityData/" + f"{entity_id}.ttl?flavor=dump" + ) + request = urllib.request.Request( + url, headers={"User-Agent": USER_AGENT} + ) + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read() + final_url = response.url + match = re.search(r"EntityData/(Q\d+)", final_url) + redirected_to = None + if match and match.group(1) != entity_id: + redirected_to = match.group(1) + return body, redirected_to + + def canonical_graph(self, ttl_bytes, entity_id, munge_script, keep_dir): + """ + Parse the canonical TTL and normalize it for comparison. With a munge + script, the ORIGINAL data must be munged (the script needs the + document node to detect the entity, and itself grafts the version and + modification date onto the entity, drops the document node, and + computes the entity-level counters). Without one, the same + normalization is done manually, minus the counters (see + `EXCLUDED_ENTITY_PREDICATES`). Returns `(graph, version)`. + """ + from rdflib import Graph, URIRef + + if munge_script is not None: + graph = self.munge(ttl_bytes, munge_script, keep_dir) + version = None + for o in graph.objects( + URIRef(f"{WD}{entity_id}"), URIRef(f"{SCHEMA}version") + ): + version = str(o) + return graph, version + + graph = Graph() + graph.parse(data=ttl_bytes, format="turtle") + entity = URIRef(f"{WD}{entity_id}") + doc_nodes = set( + s for s in graph.subjects() if "Special:EntityData" in str(s) + ) + version = None + for doc in doc_nodes: + for p, o in list(graph.predicate_objects(doc)): + if str(p) in (f"{SCHEMA}version", f"{SCHEMA}dateModified"): + graph.add((entity, p, o)) + if str(p) == f"{SCHEMA}version": + version = str(o) + graph.remove((doc, p, o)) + return graph, version + + def munge(self, ttl_bytes, munge_script, keep_dir): + """ + Run the given `munge.sh` on the given canonical TTL and return the + parsed result. + """ + from rdflib import Graph + + workdir = Path(tempfile.mkdtemp(prefix="qlever-check-sync.")) + try: + input_path = workdir / "input.ttl" + input_path.write_bytes(ttl_bytes) + result = subprocess.run( + [ + str(munge_script), + "-f", + str(input_path), + "-d", + str(workdir), + "-c", + "150000000", + "--", + "--skolemize", + ], + capture_output=True, + text=True, + ) + output_path = workdir / "wikidump-000000001.ttl.gz" + if result.returncode != 0 or not output_path.exists(): + raise Exception( + f"munge.sh failed (exit code {result.returncode}):" + f" {result.stderr.strip()[-500:]}" + ) + munged = Graph() + with gzip.open(output_path, "rt") as f: + munged.parse(data=f.read(), format="turtle") + return munged + finally: + if keep_dir is not None: + shutil.copytree( + workdir, keep_dir / workdir.name, dirs_exist_ok=True + ) + shutil.rmtree(workdir, ignore_errors=True) + + # Sampling via the MediaWiki API. + + def mediawiki_api(self, params): + import json + + url = "https://www.wikidata.org/w/api.php?" + urllib.parse.urlencode( + {**params, "format": "json"} + ) + request = urllib.request.Request( + url, headers={"User-Agent": USER_AGENT} + ) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + + def sample_entities(self, num_entities, recent_fraction, seed): + rng = random.Random(seed) + num_recent = round(num_entities * recent_fraction) + num_uniform = num_entities - num_recent + entities = [] + if num_recent > 0: + result = self.mediawiki_api( + { + "action": "query", + "list": "recentchanges", + "rcnamespace": "0", + "rctype": "edit", + "rclimit": "500", + } + ) + titles = sorted( + set( + rc["title"] + for rc in result["query"]["recentchanges"] + if re.fullmatch(r"Q\d+", rc["title"]) + ) + ) + entities += rng.sample(titles, min(num_recent, len(titles))) + while len(entities) < num_recent + num_uniform: + result = self.mediawiki_api( + { + "action": "query", + "list": "random", + "rnnamespace": "0", + "rnlimit": str( + min(20, num_recent + num_uniform - len(entities)) + ), + } + ) + entities += [ + r["title"] + for r in result["query"]["random"] + if re.fullmatch(r"Q\d+", r["title"]) + and r["title"] not in entities + ] + time.sleep(1) + return entities + + # Comparison. + + def canonical_term(self, term): + """ + Return the N-Triples form of the term, with numeric literals in a + canonical form. This is needed because a numeric literal is stored by + the index as a value with limited precision, not as a lexical form: + `"+190"^^xsd:decimal` is exported as `"190"^^xsd:decimal`, + `"25.9816937839249"^^xsd:decimal` as `"25.98169378392"^^xsd:decimal`, + and a decimal can even come back as `xsd:double`. Integers are kept + exact; all other numbers are rounded to 12 significant digits, with a + unified datatype marker. + """ + from decimal import Decimal, InvalidOperation + + from rdflib import Literal + + numeric_datatypes = { + "http://www.w3.org/2001/XMLSchema#decimal", + "http://www.w3.org/2001/XMLSchema#integer", + "http://www.w3.org/2001/XMLSchema#double", + "http://www.w3.org/2001/XMLSchema#float", + } + if ( + isinstance(term, Literal) + and term.datatype is not None + and str(term.datatype) in numeric_datatypes + ): + try: + value = Decimal(str(term)) + if value == value.to_integral_value() and abs(value) < 2**60: + lexical = str(int(value)) + else: + lexical = format(float(value), ".12g") + return f'"{lexical}"^^NUM' + except (InvalidOperation, OverflowError): + pass + return term.n3() + + def normalize_triples(self, graph, entity_id, exclude_counters): + """ + Return the set of N-Triples lines of the graph. Without munging, the + entity-level counter triples are excluded (see + `EXCLUDED_ENTITY_PREDICATES`); with munging, the munge script computes + them on the canonical side, so they are compared like all others. + """ + from rdflib import Literal + + entity = f"{WD}{entity_id}" + lines = set() + for s, p, o in graph: + if ( + exclude_counters + and str(s) == entity + and str(p) in EXCLUDED_ENTITY_PREDICATES + ): + continue + if str(p) == RDF_TYPE and str(o) in EXCLUDED_TYPE_OBJECTS: + continue + if str(p) in EXCLUDED_PREDICATES: + continue + if str(p) in GEO_COMPONENT_PREDICATES and isinstance(o, Literal): + object_string = f'"{round(float(str(o)), 6)}"^^GEO' + elif ( + isinstance(o, Literal) + and o.datatype is not None + and str(o.datatype) == WKT_DATATYPE + ): + object_string = self.canonical_wkt(str(o)) + else: + object_string = self.canonical_term(o) + lines.add(f"{s.n3()} {p.n3()} {object_string}") + return lines + + def canonical_wkt(self, wkt): + """ + Return a canonical form of the given WKT literal, with the keyword in + uppercase and the coordinates rounded to 6 decimal places (the + precision of the fixed-precision encoding used by QLever). + """ + + def round_number(match): + return str(round(float(match.group(0)), 6)) + + return re.sub(r"-?\d+(\.\d+)?", round_number, wkt.strip().upper()) + + def qlever_version(self, graph, entity_id): + from rdflib import URIRef + + for o in graph.objects( + URIRef(f"{WD}{entity_id}"), URIRef(f"{SCHEMA}version") + ): + return str(o) + return None + + def check_entity(self, entity_id, endpoint, munge_script, keep_dir): + """ + Check a single entity. Returns one of `match`, `divergent`, + `undecidable`, `redirect`, or `error`. + """ + try: + ttl_bytes, redirected_to = self.fetch_canonical(entity_id) + if redirected_to is not None: + log.info(f"{entity_id}: redirect to {redirected_to}, skipped") + return "redirect" + qlever_graph = self.qlever_entity_graph(endpoint, entity_id) + canonical_graph, canonical_version = self.canonical_graph( + ttl_bytes, entity_id, munge_script, keep_dir + ) + qlever_version = self.qlever_version(qlever_graph, entity_id) + if canonical_version is None or qlever_version is None: + log.warning( + f"{entity_id}: could not determine version" + f" (canonical: {canonical_version}," + f" endpoint: {qlever_version})" + ) + return "error" + if canonical_version != qlever_version: + log.info( + f"{entity_id}: version mismatch (canonical:" + f" {canonical_version}, endpoint: {qlever_version})," + f" edited since the endpoint's stream position" + ) + return "undecidable" + exclude_counters = munge_script is None + canonical = self.normalize_triples( + canonical_graph, entity_id, exclude_counters + ) + qlever = self.normalize_triples( + qlever_graph, entity_id, exclude_counters + ) + missing = canonical - qlever + extra = qlever - canonical + if not missing and not extra: + log.info( + f"{entity_id}: exact match at version" + f" {qlever_version} ({len(qlever):,} triples)" + ) + return "match" + log.error( + f"{entity_id}: DIVERGENT at version {qlever_version}" + f" ({len(missing)} triples missing on the endpoint," + f" {len(extra)} extra)" + ) + for line in sorted(missing)[:5]: + log.error(f" missing: {line}") + for line in sorted(extra)[:5]: + log.error(f" extra: {line}") + return "divergent" + except Exception as e: + log.warning(f"{entity_id}: check failed ({e})") + return "error" + + def execute(self, args) -> bool: + endpoint = ( + args.sparql_endpoint + if args.sparql_endpoint + else f"http://{args.host_name}:{args.port}" + ) + munge_scripts = sorted(glob.glob("service-*/munge.sh")) + if args.munge == "yes" and not munge_scripts: + log.error( + "`--munge yes` was given, but no `service-*/munge.sh` was" + " found in the current directory" + ) + return False + munge_script = None + if munge_scripts and args.munge in ("auto", "yes"): + munge_script = Path(munge_scripts[-1]).resolve() + description = ( + f"Check entities on {endpoint} against wikidata.org" + f" (munge: {munge_script or 'no'})" + ) + self.show(description, only_show=args.show) + if args.show: + return True + + keep_dir = Path.cwd() if args.keep_files else None + if args.entities: + entities = args.entities.split(",") + else: + log.info( + f"Sampling {args.num_entities} entities" + f" ({args.recent_fraction:.0%} recently edited," + f" seed {args.seed}) ..." + ) + entities = self.sample_entities( + args.num_entities, args.recent_fraction, args.seed + ) + log.info(f"Entities: {', '.join(entities)}") + + outcomes = {} + for entity_id in entities: + outcome = self.check_entity( + entity_id, endpoint, munge_script, keep_dir + ) + if outcome == "undecidable": + time.sleep(5) + log.info(f"{entity_id}: retrying once ...") + outcome = self.check_entity( + entity_id, endpoint, munge_script, keep_dir + ) + outcomes[entity_id] = outcome + time.sleep(1) + + counts = { + status: sum(1 for o in outcomes.values() if o == status) + for status in ( + "match", + "divergent", + "undecidable", + "redirect", + "error", + ) + } + log.info("") + log.info( + f"Result: {counts['match']} exact matches," + f" {counts['divergent']} divergent," + f" {counts['undecidable']} undecidable (edited during check)," + f" {counts['redirect']} redirects skipped," + f" {counts['error']} errors" + ) + if counts["divergent"] > 0: + divergent = [e for e, o in outcomes.items() if o == "divergent"] + log.error(f"Endpoint DIVERGES from wikidata.org: {divergent}") + return False + return True From 39f5ea3ea0ba0f1a347fa154603a012c74288f24 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Tue, 28 Jul 2026 10:06:07 +0200 Subject: [PATCH 02/16] Address the review comments and add unit tests Move all imports to the top of the module (as in the other commands), deduplicate the version extraction into `entity_version`, correct the help text of `--sparql-endpoint`, validate `--entities` and `--recent-fraction`, add a timeout for `munge.sh`, and add unit tests for the deterministic parts (normalization, numeric and geographic canonicalization, version extraction). --- .../commands/check_sync_with_wikidata.py | 47 +++--- .../test_check_sync_with_wikidata_methods.py | 158 ++++++++++++++++++ 2 files changed, 179 insertions(+), 26 deletions(-) create mode 100644 test/qlever/commands/test_check_sync_with_wikidata_methods.py diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 4d1276b2..5bda8836 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -2,6 +2,7 @@ import glob import gzip +import json import random import re import shutil @@ -10,8 +11,11 @@ import time import urllib.parse import urllib.request +from decimal import Decimal, InvalidOperation from pathlib import Path +from rdflib import Graph, Literal, URIRef + from qlever.command import QleverCommand from qlever.log import log @@ -75,7 +79,8 @@ def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: def additional_arguments(self, subparser) -> None: subparser.add_argument( "--sparql-endpoint", - help="URL of the QLever server, default is {host_name}:{port}", + help="URL of the QLever server," + " default is http://{host_name}:{port}", ) subparser.add_argument( "--entities", @@ -140,8 +145,6 @@ def qlever_entity_graph(self, endpoint, entity_id): from the entity (plus their references and values), and the sitelink article blocks. """ - from rdflib import Graph - e = f"<{WD}{entity_id}>" # NOTE: Statement IRIs of old statements contain the entity ID in # lowercase. @@ -210,16 +213,9 @@ def canonical_graph(self, ttl_bytes, entity_id, munge_script, keep_dir): normalization is done manually, minus the counters (see `EXCLUDED_ENTITY_PREDICATES`). Returns `(graph, version)`. """ - from rdflib import Graph, URIRef - if munge_script is not None: graph = self.munge(ttl_bytes, munge_script, keep_dir) - version = None - for o in graph.objects( - URIRef(f"{WD}{entity_id}"), URIRef(f"{SCHEMA}version") - ): - version = str(o) - return graph, version + return graph, self.entity_version(graph, entity_id) graph = Graph() graph.parse(data=ttl_bytes, format="turtle") @@ -242,8 +238,6 @@ def munge(self, ttl_bytes, munge_script, keep_dir): Run the given `munge.sh` on the given canonical TTL and return the parsed result. """ - from rdflib import Graph - workdir = Path(tempfile.mkdtemp(prefix="qlever-check-sync.")) try: input_path = workdir / "input.ttl" @@ -262,6 +256,7 @@ def munge(self, ttl_bytes, munge_script, keep_dir): ], capture_output=True, text=True, + timeout=300, ) output_path = workdir / "wikidump-000000001.ttl.gz" if result.returncode != 0 or not output_path.exists(): @@ -283,8 +278,6 @@ def munge(self, ttl_bytes, munge_script, keep_dir): # Sampling via the MediaWiki API. def mediawiki_api(self, params): - import json - url = "https://www.wikidata.org/w/api.php?" + urllib.parse.urlencode( {**params, "format": "json"} ) @@ -350,10 +343,6 @@ def canonical_term(self, term): exact; all other numbers are rounded to 12 significant digits, with a unified datatype marker. """ - from decimal import Decimal, InvalidOperation - - from rdflib import Literal - numeric_datatypes = { "http://www.w3.org/2001/XMLSchema#decimal", "http://www.w3.org/2001/XMLSchema#integer", @@ -383,8 +372,6 @@ def normalize_triples(self, graph, entity_id, exclude_counters): `EXCLUDED_ENTITY_PREDICATES`); with munging, the munge script computes them on the canonical side, so they are compared like all others. """ - from rdflib import Literal - entity = f"{WD}{entity_id}" lines = set() for s, p, o in graph: @@ -423,9 +410,10 @@ def round_number(match): return re.sub(r"-?\d+(\.\d+)?", round_number, wkt.strip().upper()) - def qlever_version(self, graph, entity_id): - from rdflib import URIRef - + def entity_version(self, graph, entity_id): + """ + Return the `schema:version` of the given entity in the given graph. + """ for o in graph.objects( URIRef(f"{WD}{entity_id}"), URIRef(f"{SCHEMA}version") ): @@ -446,7 +434,7 @@ def check_entity(self, entity_id, endpoint, munge_script, keep_dir): canonical_graph, canonical_version = self.canonical_graph( ttl_bytes, entity_id, munge_script, keep_dir ) - qlever_version = self.qlever_version(qlever_graph, entity_id) + qlever_version = self.entity_version(qlever_graph, entity_id) if canonical_version is None or qlever_version is None: log.warning( f"{entity_id}: could not determine version" @@ -515,8 +503,15 @@ def execute(self, args) -> bool: return True keep_dir = Path.cwd() if args.keep_files else None + if not 0.0 <= args.recent_fraction <= 1.0: + log.error("`--recent-fraction` must be between 0.0 and 1.0") + return False if args.entities: - entities = args.entities.split(",") + entities = [e.strip() for e in args.entities.split(",")] + invalid = [e for e in entities if not re.fullmatch(r"Q\d+", e)] + if invalid: + log.error(f"Invalid entity IDs: {invalid}") + return False else: log.info( f"Sampling {args.num_entities} entities" diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py new file mode 100644 index 00000000..c37704e0 --- /dev/null +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -0,0 +1,158 @@ +import unittest + +from rdflib import Graph, Literal, URIRef + +from qlever.commands.check_sync_with_wikidata import ( + CheckSyncWithWikidataCommand, +) + +XSD = "http://www.w3.org/2001/XMLSchema#" +WD = "http://www.wikidata.org/entity/" +WIKIBASE = "http://wikiba.se/ontology#" + + +def literal(lexical, datatype): + return Literal(lexical, datatype=URIRef(f"{XSD}{datatype}")) + + +class TestCheckSyncWithWikidataCommand(unittest.TestCase): + def setUp(self): + self.command = CheckSyncWithWikidataCommand() + + def test_description(self): + self.assertEqual( + self.command.description(), + "Check that entities on this endpoint are exactly in sync with" + " wikidata.org, by comparing against `Special:EntityData`", + ) + + def test_should_have_qleverfile(self): + self.assertTrue(self.command.should_have_qleverfile()) + + def test_relevant_qleverfile_arguments(self): + self.assertEqual( + self.command.relevant_qleverfile_arguments(), + {"server": ["host_name", "port"]}, + ) + + def test_canonical_term_numeric(self): + # The lexical form of a numeric literal is not preserved by the + # index, so numbers are compared by value: leading `+` and trailing + # zeros disappear, and the datatype is unified. + self.assertEqual( + self.command.canonical_term(literal("+190", "decimal")), + '"190"^^NUM', + ) + self.assertEqual( + self.command.canonical_term(literal("190.0", "double")), + '"190"^^NUM', + ) + # Values are rounded to 12 significant digits (the index stores + # decimals with limited precision). + self.assertEqual( + self.command.canonical_term( + literal("25.9816937839249", "decimal") + ), + self.command.canonical_term(literal("25.98169378392", "decimal")), + ) + # Large integers are kept exact. + self.assertEqual( + self.command.canonical_term( + literal("123456789012345678", "integer") + ), + '"123456789012345678"^^NUM', + ) + + def test_canonical_term_non_numeric(self): + self.assertEqual( + self.command.canonical_term(Literal("hello", lang="en")), + '"hello"@en', + ) + self.assertEqual( + self.command.canonical_term(URIRef(f"{WD}Q42")), + f"<{WD}Q42>", + ) + + def test_canonical_wkt(self): + # Coordinates are rounded to 6 decimal places and the keyword is + # uppercased, matching the fixed-precision encoding of QLever. + self.assertEqual( + self.command.canonical_wkt( + "Point(-0.14544444444444 51.566527777778)" + ), + self.command.canonical_wkt("POINT(-0.145444 51.566528)"), + ) + + def test_normalize_triples_exclusions(self): + graph = Graph() + entity = URIRef(f"{WD}Q42") + # An ordinary triple, an entity-level counter, a reference-type + # triple, and a normalized-quantity link. + graph.add( + ( + entity, + URIRef(f"{WD.replace('entity/', 'prop/direct/')}P31"), + URIRef(f"{WD}Q5"), + ) + ) + graph.add( + (entity, URIRef(f"{WIKIBASE}sitelinks"), literal("5", "integer")) + ) + graph.add( + ( + URIRef("http://www.wikidata.org/reference/abc"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef(f"{WIKIBASE}Reference"), + ) + ) + graph.add( + ( + URIRef("http://www.wikidata.org/value/abc"), + URIRef(f"{WIKIBASE}quantityNormalized"), + URIRef("http://www.wikidata.org/value/def"), + ) + ) + with_counters = self.command.normalize_triples(graph, "Q42", False) + without_counters = self.command.normalize_triples(graph, "Q42", True) + # The reference type and the normalized-quantity link are always + # excluded; the counter only with `exclude_counters`. + self.assertEqual(len(with_counters), 2) + self.assertEqual(len(without_counters), 1) + + def test_normalize_triples_geo(self): + # The two lexical forms of the same coordinate (canonical vs. the + # export of the fixed-precision encoding) must compare equal. + canonical, qlever = Graph(), Graph() + value = URIRef("http://www.wikidata.org/value/abc") + latitude = URIRef(f"{WIKIBASE}geoLatitude") + canonical.add((value, latitude, literal("51.566527777778", "double"))) + qlever.add((value, latitude, literal("51.56652777778", "decimal"))) + self.assertEqual( + self.command.normalize_triples(canonical, "Q42", False), + self.command.normalize_triples(qlever, "Q42", False), + ) + + def test_canonical_graph_without_munging(self): + # The version and modification date are grafted from the document + # node onto the entity, and the document node is dropped. + ttl = f""" + @prefix schema: . + @prefix wd: <{WD}> . + @prefix xsd: <{XSD}> . + + schema:about wd:Q42 ; + schema:version "123"^^xsd:integer ; + schema:dateModified "2026-07-28T00:00:00Z"^^xsd:dateTime . + wd:Q42 schema:name "Douglas Adams"@en . + """ + graph, version = self.command.canonical_graph( + ttl.encode(), "Q42", None, None + ) + self.assertEqual(version, "123") + self.assertEqual(self.command.entity_version(graph, "Q42"), "123") + subjects = set(str(s) for s in graph.subjects()) + self.assertEqual(subjects, {f"{WD}Q42"}) + + +if __name__ == "__main__": + unittest.main() From cd0a6c7d73716d664a09a6f7fc285f5871d7d40c Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Tue, 28 Jul 2026 10:36:38 +0200 Subject: [PATCH 03/16] Compare numbers at 10 significant digits A 100-entity random-sample run found a value where the export of the index differs from the canonical lexical form in the 12th significant digit (168.73846826 vs 168.738468261): the encoding used by the index rounds slightly differently than IEEE string parsing. Round to 10 significant digits, which is safely below such differences and still far more precision than any real divergence would survive. --- src/qlever/commands/check_sync_with_wikidata.py | 8 +++++--- .../commands/test_check_sync_with_wikidata_methods.py | 9 +++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 5bda8836..cec5bad6 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -340,8 +340,10 @@ def canonical_term(self, term): `"+190"^^xsd:decimal` is exported as `"190"^^xsd:decimal`, `"25.9816937839249"^^xsd:decimal` as `"25.98169378392"^^xsd:decimal`, and a decimal can even come back as `xsd:double`. Integers are kept - exact; all other numbers are rounded to 12 significant digits, with a - unified datatype marker. + exact; all other numbers are rounded to 10 significant digits, with a + unified datatype marker (the encoding used by the index rounds + slightly differently than IEEE string parsing, with differences + observed in the 12th significant digit). """ numeric_datatypes = { "http://www.w3.org/2001/XMLSchema#decimal", @@ -359,7 +361,7 @@ def canonical_term(self, term): if value == value.to_integral_value() and abs(value) < 2**60: lexical = str(int(value)) else: - lexical = format(float(value), ".12g") + lexical = format(float(value), ".10g") return f'"{lexical}"^^NUM' except (InvalidOperation, OverflowError): pass diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py index c37704e0..c51f5a0d 100644 --- a/test/qlever/commands/test_check_sync_with_wikidata_methods.py +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -47,14 +47,19 @@ def test_canonical_term_numeric(self): self.command.canonical_term(literal("190.0", "double")), '"190"^^NUM', ) - # Values are rounded to 12 significant digits (the index stores - # decimals with limited precision). + # Values are rounded to 10 significant digits (the index stores + # decimals with limited precision and rounds slightly differently + # than IEEE string parsing). self.assertEqual( self.command.canonical_term( literal("25.9816937839249", "decimal") ), self.command.canonical_term(literal("25.98169378392", "decimal")), ) + self.assertEqual( + self.command.canonical_term(literal("168.73846826", "decimal")), + self.command.canonical_term(literal("168.738468261", "decimal")), + ) # Large integers are kept exact. self.assertEqual( self.command.canonical_term( From 3f1194ec44894faf2d313fd4118986da89c91b09 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Tue, 28 Jul 2026 11:05:07 +0200 Subject: [PATCH 04/16] Munge in batches (new option `--batch-size`) Each run of `munge.sh` starts a JVM, which took 3-4 of the ~6 seconds per checked entity. Munge a whole batch (default: 50 entities) in one run instead: the canonical documents are downloaded and the endpoint is queried pairwise as before (the snapshot semantics of the version gate do not change), the concatenation of the downloaded documents is munged in one go, and `extract_document` then reconstructs the entity boundaries from the munged output, mirroring exactly the queries used on the endpoint side. This brings the cost down to ~2 seconds per entity, dominated by the polite pacing of the downloads. --- .../commands/check_sync_with_wikidata.py | 265 +++++++++++++----- .../test_check_sync_with_wikidata_methods.py | 49 +++- 2 files changed, 239 insertions(+), 75 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index cec5bad6..e43ed6e2 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -26,6 +26,8 @@ SCHEMA = "http://schema.org/" WIKIBASE = "http://wikiba.se/ontology#" STATEMENT = "http://www.wikidata.org/entity/statement/" +REFERENCE = "http://www.wikidata.org/reference/" +VALUE = "http://www.wikidata.org/value/" # Entity-level counter triples that are contained in the full dump (and hence # in the index), but NOT in the output of `Special:EntityData`. They are @@ -107,6 +109,14 @@ def additional_arguments(self, subparser) -> None: default=42, help="Seed for the random sample (default: 42)", ) + subparser.add_argument( + "--batch-size", + type=int, + default=50, + help="Number of entities munged together in one run of" + " `munge.sh` (default: 50); this amortizes the JVM startup," + " which dominates the cost of checking a single entity", + ) subparser.add_argument( "--munge", choices=["auto", "yes", "no"], @@ -203,20 +213,15 @@ def fetch_canonical(self, entity_id): redirected_to = match.group(1) return body, redirected_to - def canonical_graph(self, ttl_bytes, entity_id, munge_script, keep_dir): + def canonical_graph(self, ttl_bytes, entity_id): """ - Parse the canonical TTL and normalize it for comparison. With a munge - script, the ORIGINAL data must be munged (the script needs the - document node to detect the entity, and itself grafts the version and - modification date onto the entity, drops the document node, and - computes the entity-level counters). Without one, the same - normalization is done manually, minus the counters (see + Parse the canonical TTL and normalize it like `munge.sh` would, for + a comparison WITHOUT munging: graft the version and modification date + from the document node onto the entity and drop the document node. + Unlike `munge.sh`, this cannot compute the entity-level counters, + which is why they are excluded from the comparison in this mode (see `EXCLUDED_ENTITY_PREDICATES`). Returns `(graph, version)`. """ - if munge_script is not None: - graph = self.munge(ttl_bytes, munge_script, keep_dir) - return graph, self.entity_version(graph, entity_id) - graph = Graph() graph.parse(data=ttl_bytes, format="turtle") entity = URIRef(f"{WD}{entity_id}") @@ -233,6 +238,58 @@ def canonical_graph(self, ttl_bytes, entity_id, munge_script, keep_dir): graph.remove((doc, p, o)) return graph, version + def extract_document(self, graph, entity_id): + """ + Extract the document of the given entity from the given graph: the + triples with the entity as subject, the statement nodes of the + entity (recognized by their IRI prefix), the references and values + reachable from those statements, the sitelink article blocks, and + the wiki metadata. This mirrors exactly the queries of + `qlever_entity_graph`, so that the two sides of the comparison cover + the same universe. It is what makes munging in batches possible: the + munged output of a batch is one graph without entity boundaries, and + this reconstructs them (references and values are shared between + entities and are assigned to every entity that reaches them, on both + sides). + """ + entity = URIRef(f"{WD}{entity_id}") + statement_prefixes = ( + f"{STATEMENT}{entity_id}-", + f"{STATEMENT}{entity_id.lower()}-", + ) + document = Graph() + statements = set() + for p, o in graph.predicate_objects(entity): + document.add((entity, p, o)) + if isinstance(o, URIRef) and str(o).startswith(statement_prefixes): + statements.add(o) + references_and_values = set() + for statement in statements: + for p, o in graph.predicate_objects(statement): + document.add((statement, p, o)) + if isinstance(o, URIRef) and str(o).startswith( + (REFERENCE, VALUE) + ): + references_and_values.add(o) + # References can point to values. + for node in list(references_and_values): + for _, o in graph.predicate_objects(node): + if isinstance(o, URIRef) and str(o).startswith(VALUE): + references_and_values.add(o) + for node in references_and_values: + for p, o in graph.predicate_objects(node): + document.add((node, p, o)) + wikis = set() + for article in graph.subjects(URIRef(f"{SCHEMA}about"), entity): + for p, o in graph.predicate_objects(article): + document.add((article, p, o)) + if str(p) == f"{SCHEMA}isPartOf": + wikis.add(o) + for wiki in wikis: + for p, o in graph.predicate_objects(wiki): + document.add((wiki, p, o)) + return document + def munge(self, ttl_bytes, munge_script, keep_dir): """ Run the given `munge.sh` on the given canonical TTL and return the @@ -422,63 +479,116 @@ def entity_version(self, graph, entity_id): return str(o) return None - def check_entity(self, entity_id, endpoint, munge_script, keep_dir): + def compare_entity( + self, entity_id, qlever_graph, canonical_document, exclude_counters + ): """ - Check a single entity. Returns one of `match`, `divergent`, - `undecidable`, `redirect`, or `error`. + Compare the two documents of the given entity. Returns one of + `match`, `divergent`, `undecidable`, or `error`. """ - try: - ttl_bytes, redirected_to = self.fetch_canonical(entity_id) - if redirected_to is not None: - log.info(f"{entity_id}: redirect to {redirected_to}, skipped") - return "redirect" - qlever_graph = self.qlever_entity_graph(endpoint, entity_id) - canonical_graph, canonical_version = self.canonical_graph( - ttl_bytes, entity_id, munge_script, keep_dir + canonical_version = self.entity_version(canonical_document, entity_id) + qlever_version = self.entity_version(qlever_graph, entity_id) + if canonical_version is None or qlever_version is None: + log.warning( + f"{entity_id}: could not determine version" + f" (canonical: {canonical_version}," + f" endpoint: {qlever_version})" ) - qlever_version = self.entity_version(qlever_graph, entity_id) - if canonical_version is None or qlever_version is None: - log.warning( - f"{entity_id}: could not determine version" - f" (canonical: {canonical_version}," - f" endpoint: {qlever_version})" - ) - return "error" - if canonical_version != qlever_version: - log.info( - f"{entity_id}: version mismatch (canonical:" - f" {canonical_version}, endpoint: {qlever_version})," - f" edited since the endpoint's stream position" - ) - return "undecidable" - exclude_counters = munge_script is None - canonical = self.normalize_triples( - canonical_graph, entity_id, exclude_counters + return "error" + if canonical_version != qlever_version: + log.info( + f"{entity_id}: version mismatch (canonical:" + f" {canonical_version}, endpoint: {qlever_version})," + f" edited since the endpoint's stream position" ) - qlever = self.normalize_triples( - qlever_graph, entity_id, exclude_counters + return "undecidable" + canonical = self.normalize_triples( + canonical_document, entity_id, exclude_counters + ) + qlever = self.normalize_triples( + qlever_graph, entity_id, exclude_counters + ) + missing = canonical - qlever + extra = qlever - canonical + if not missing and not extra: + log.info( + f"{entity_id}: exact match at version" + f" {qlever_version} ({len(qlever):,} triples)" ) - missing = canonical - qlever - extra = qlever - canonical - if not missing and not extra: - log.info( - f"{entity_id}: exact match at version" - f" {qlever_version} ({len(qlever):,} triples)" + return "match" + log.error( + f"{entity_id}: DIVERGENT at version {qlever_version}" + f" ({len(missing)} triples missing on the endpoint," + f" {len(extra)} extra)" + ) + for line in sorted(missing)[:5]: + log.error(f" missing: {line}") + for line in sorted(extra)[:5]: + log.error(f" extra: {line}") + return "divergent" + + def check_batch(self, batch, endpoint, munge_script, keep_dir): + """ + Check a batch of entities: download the canonical data and query the + endpoint pairwise (so that the version gate has the best chance), + then munge the whole batch in ONE run of `munge.sh`, and compare + entity by entity. Returns a dict from entity ID to outcome. + """ + outcomes = {} + snapshots = [] + for entity_id in batch: + try: + ttl_bytes, redirected_to = self.fetch_canonical(entity_id) + if redirected_to is not None: + log.info( + f"{entity_id}: redirect to {redirected_to}, skipped" + ) + outcomes[entity_id] = "redirect" + else: + qlever_graph = self.qlever_entity_graph( + endpoint, entity_id + ) + snapshots.append((entity_id, ttl_bytes, qlever_graph)) + except Exception as e: + log.warning(f"{entity_id}: check failed ({e})") + outcomes[entity_id] = "error" + time.sleep(1) + batch_graph = None + if munge_script is not None and snapshots: + try: + batch_graph = self.munge( + b"".join(ttl for _, ttl, _ in snapshots), + munge_script, + keep_dir, ) - return "match" - log.error( - f"{entity_id}: DIVERGENT at version {qlever_version}" - f" ({len(missing)} triples missing on the endpoint," - f" {len(extra)} extra)" - ) - for line in sorted(missing)[:5]: - log.error(f" missing: {line}") - for line in sorted(extra)[:5]: - log.error(f" extra: {line}") - return "divergent" - except Exception as e: - log.warning(f"{entity_id}: check failed ({e})") - return "error" + except Exception as e: + log.warning(f"Munging the batch failed ({e})") + for entity_id, _, _ in snapshots: + outcomes[entity_id] = "error" + return outcomes + for entity_id, ttl_bytes, qlever_graph in snapshots: + try: + if batch_graph is not None: + canonical_document = self.extract_document( + batch_graph, entity_id + ) + exclude_counters = False + else: + graph, _ = self.canonical_graph(ttl_bytes, entity_id) + canonical_document = self.extract_document( + graph, entity_id + ) + exclude_counters = True + outcomes[entity_id] = self.compare_entity( + entity_id, + qlever_graph, + canonical_document, + exclude_counters, + ) + except Exception as e: + log.warning(f"{entity_id}: check failed ({e})") + outcomes[entity_id] = "error" + return outcomes def execute(self, args) -> bool: endpoint = ( @@ -525,19 +635,30 @@ def execute(self, args) -> bool: ) log.info(f"Entities: {', '.join(entities)}") + if args.batch_size < 1: + log.error("`--batch-size` must be at least 1") + return False + + def batches(entity_ids): + for i in range(0, len(entity_ids), args.batch_size): + yield entity_ids[i : i + args.batch_size] + outcomes = {} - for entity_id in entities: - outcome = self.check_entity( - entity_id, endpoint, munge_script, keep_dir + for batch in batches(entities): + outcomes.update( + self.check_batch(batch, endpoint, munge_script, keep_dir) + ) + retry = [e for e, o in outcomes.items() if o == "undecidable"] + if retry: + log.info( + f"Retrying {len(retry)} entities that were edited" + f" during the check ..." ) - if outcome == "undecidable": - time.sleep(5) - log.info(f"{entity_id}: retrying once ...") - outcome = self.check_entity( - entity_id, endpoint, munge_script, keep_dir + time.sleep(5) + for batch in batches(retry): + outcomes.update( + self.check_batch(batch, endpoint, munge_script, keep_dir) ) - outcomes[entity_id] = outcome - time.sleep(1) counts = { status: sum(1 for o in outcomes.values() if o == status) diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py index c51f5a0d..ee471f57 100644 --- a/test/qlever/commands/test_check_sync_with_wikidata_methods.py +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -150,14 +150,57 @@ def test_canonical_graph_without_munging(self): schema:dateModified "2026-07-28T00:00:00Z"^^xsd:dateTime . wd:Q42 schema:name "Douglas Adams"@en . """ - graph, version = self.command.canonical_graph( - ttl.encode(), "Q42", None, None - ) + graph, version = self.command.canonical_graph(ttl.encode(), "Q42") self.assertEqual(version, "123") self.assertEqual(self.command.entity_version(graph, "Q42"), "123") subjects = set(str(s) for s in graph.subjects()) self.assertEqual(subjects, {f"{WD}Q42"}) + def test_extract_document(self): + # The document of Q42 consists of its subject triples, its statement + # nodes (also with the lowercase IRIs of old statements), the + # references and values reachable from them, and the sitelink + # article blocks with their wiki metadata; the statement of the + # OTHER entity Q43 does not belong to it. + turtle = f""" + @prefix schema: . + @prefix wd: <{WD}> . + @prefix wds: . + @prefix wdref: . + @prefix wdv: . + @prefix p: . + @prefix prov: . + wd:Q42 p:P31 wds:Q42-aaa , wds:q42-bbb . + wds:Q42-aaa prov:wasDerivedFrom wdref:ref1 . + wds:q42-bbb p:P2 wdv:value1 . + wdref:ref1 p:P3 wdv:value2 . + wdv:value1 p:P4 "x" . + wdv:value2 p:P5 "y" . + schema:about wd:Q42 ; + schema:isPartOf . + p:P6 "wiki" . + wd:Q43 p:P31 wds:Q43-ccc . + wds:Q43-ccc p:P7 "other" . + """ + graph = Graph() + graph.parse(data=turtle, format="turtle") + document = self.command.extract_document(graph, "Q42") + subjects = set(str(s) for s in document.subjects()) + self.assertEqual( + subjects, + { + f"{WD}Q42", + "http://www.wikidata.org/entity/statement/Q42-aaa", + "http://www.wikidata.org/entity/statement/q42-bbb", + "http://www.wikidata.org/reference/ref1", + "http://www.wikidata.org/value/value1", + "http://www.wikidata.org/value/value2", + "https://en.wikipedia.org/wiki/A", + "https://en.wikipedia.org/", + }, + ) + self.assertEqual(len(document), 10) + if __name__ == "__main__": unittest.main() From ced152510a5830c347ca726254496ccbf0ee9be1 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Tue, 28 Jul 2026 11:23:51 +0200 Subject: [PATCH 05/16] Compare geographic coordinates at 5 decimal places The 1000-entity run found a coordinate where the export of the index differs from the canonical form by 1e-6 degrees (5.483421 came back as 5.48342), so rounding to 6 decimal places is not enough. --- src/qlever/commands/check_sync_with_wikidata.py | 13 +++++++------ .../test_check_sync_with_wikidata_methods.py | 6 +++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index e43ed6e2..7085eb87 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -48,8 +48,9 @@ EXCLUDED_PREDICATES = {f"{WIKIBASE}quantityNormalized"} # Geographic coordinates are stored by QLever in a fixed-precision encoding -# (roughly 1e-6 degrees) and exported in a normalized form, so they are -# compared after rounding to 6 decimal places. +# and exported in a normalized form; differences up to 1e-6 degrees have been +# observed (5.483421 came back as 5.48342), so they are compared after +# rounding to 5 decimal places. GEO_COMPONENT_PREDICATES = { f"{WIKIBASE}geoLatitude", f"{WIKIBASE}geoLongitude", @@ -445,7 +446,7 @@ def normalize_triples(self, graph, entity_id, exclude_counters): if str(p) in EXCLUDED_PREDICATES: continue if str(p) in GEO_COMPONENT_PREDICATES and isinstance(o, Literal): - object_string = f'"{round(float(str(o)), 6)}"^^GEO' + object_string = f'"{round(float(str(o)), 5)}"^^GEO' elif ( isinstance(o, Literal) and o.datatype is not None @@ -460,12 +461,12 @@ def normalize_triples(self, graph, entity_id, exclude_counters): def canonical_wkt(self, wkt): """ Return a canonical form of the given WKT literal, with the keyword in - uppercase and the coordinates rounded to 6 decimal places (the - precision of the fixed-precision encoding used by QLever). + uppercase and the coordinates rounded to 5 decimal places (see the + comment at `GEO_COMPONENT_PREDICATES`). """ def round_number(match): - return str(round(float(match.group(0)), 6)) + return str(round(float(match.group(0)), 5)) return re.sub(r"-?\d+(\.\d+)?", round_number, wkt.strip().upper()) diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py index ee471f57..ca84781f 100644 --- a/test/qlever/commands/test_check_sync_with_wikidata_methods.py +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -79,7 +79,7 @@ def test_canonical_term_non_numeric(self): ) def test_canonical_wkt(self): - # Coordinates are rounded to 6 decimal places and the keyword is + # Coordinates are rounded to 5 decimal places and the keyword is # uppercased, matching the fixed-precision encoding of QLever. self.assertEqual( self.command.canonical_wkt( @@ -87,6 +87,10 @@ def test_canonical_wkt(self): ), self.command.canonical_wkt("POINT(-0.145444 51.566528)"), ) + self.assertEqual( + self.command.canonical_wkt("Point(5.483421 50.642359)"), + self.command.canonical_wkt("POINT(5.48342 50.642359)"), + ) def test_normalize_triples_exclusions(self): graph = Graph() From 13e40c49c617733f2ecb289a91983a8e0c756fdf Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Wed, 29 Jul 2026 09:48:22 +0200 Subject: [PATCH 06/16] Compare geographic values with a tolerance instead of rounding The second 1000-entity run found a coordinate at a rounding boundary (50.81588 exported as 50.81587), where rounding to any fixed number of decimal places compares unequal. Exclude the geographic values from the exact set comparison (their triples still take part via a placeholder, so a missing or extra triple is still detected exactly) and compare the values separately with a tolerance of 2e-5. --- .../commands/check_sync_with_wikidata.py | 85 ++++++++++++++----- .../test_check_sync_with_wikidata_methods.py | 53 ++++++++---- 2 files changed, 98 insertions(+), 40 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 7085eb87..03a82427 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -48,15 +48,19 @@ EXCLUDED_PREDICATES = {f"{WIKIBASE}quantityNormalized"} # Geographic coordinates are stored by QLever in a fixed-precision encoding -# and exported in a normalized form; differences up to 1e-6 degrees have been -# observed (5.483421 came back as 5.48342), so they are compared after -# rounding to 5 decimal places. +# and exported in a normalized form; differences up to 1e-5 degrees have +# been observed (5.483421 came back as 5.48342, and 50.81588 as 50.81587). +# Rounding cannot absorb such differences reliably (whatever the number of +# decimal places, an encoding error can straddle a rounding boundary), so +# the geographic values are excluded from the exact set comparison and +# compared separately with the following tolerance. GEO_COMPONENT_PREDICATES = { f"{WIKIBASE}geoLatitude", f"{WIKIBASE}geoLongitude", f"{WIKIBASE}geoPrecision", } WKT_DATATYPE = "http://www.opengis.net/ont/geosparql#wktLiteral" +GEO_TOLERANCE = 2e-5 class CheckSyncWithWikidataCommand(QleverCommand): @@ -427,13 +431,19 @@ def canonical_term(self, term): def normalize_triples(self, graph, entity_id, exclude_counters): """ - Return the set of N-Triples lines of the graph. Without munging, the - entity-level counter triples are excluded (see - `EXCLUDED_ENTITY_PREDICATES`); with munging, the munge script computes - them on the canonical side, so they are compared like all others. + Return `(lines, geo_values)`: the set of N-Triples lines of the + graph, and the geographic values, which take part in the set + comparison only via a placeholder (so that the PRESENCE of each such + triple is still compared exactly) and whose values are compared + separately with a tolerance (see `GEO_TOLERANCE`). Without munging, + the entity-level counter triples are excluded (see + `EXCLUDED_ENTITY_PREDICATES`); with munging, the munge script + computes them on the canonical side, so they are compared like all + others. """ entity = f"{WD}{entity_id}" lines = set() + geo_values = {} for s, p, o in graph: if ( exclude_counters @@ -445,30 +455,54 @@ def normalize_triples(self, graph, entity_id, exclude_counters): continue if str(p) in EXCLUDED_PREDICATES: continue - if str(p) in GEO_COMPONENT_PREDICATES and isinstance(o, Literal): - object_string = f'"{round(float(str(o)), 5)}"^^GEO' - elif ( + is_geo_component = str( + p + ) in GEO_COMPONENT_PREDICATES and isinstance(o, Literal) + is_wkt = ( isinstance(o, Literal) and o.datatype is not None and str(o.datatype) == WKT_DATATYPE - ): - object_string = self.canonical_wkt(str(o)) + ) + if is_geo_component or is_wkt: + object_string = "GEO" + numbers = [ + float(match.group(0)) + for match in re.finditer(r"-?\d+(\.\d+)?", str(o)) + ] + geo_values.setdefault(f"{s.n3()} {p.n3()}", []).append(numbers) else: object_string = self.canonical_term(o) lines.add(f"{s.n3()} {p.n3()} {object_string}") - return lines + return lines, geo_values - def canonical_wkt(self, wkt): + def geo_values_match(self, canonical_values, qlever_values): """ - Return a canonical form of the given WKT literal, with the keyword in - uppercase and the coordinates rounded to 5 decimal places (see the - comment at `GEO_COMPONENT_PREDICATES`). + Compare the two dicts of geographic values (as returned by + `normalize_triples`): for each subject and predicate, each list of + numbers on the one side must have a counterpart on the other side + whose numbers are all within `GEO_TOLERANCE`. """ + if set(canonical_values) != set(qlever_values): + return False - def round_number(match): - return str(round(float(match.group(0)), 5)) + def close(numbers_1, numbers_2): + return len(numbers_1) == len(numbers_2) and all( + abs(a - b) <= GEO_TOLERANCE + for a, b in zip(numbers_1, numbers_2) + ) - return re.sub(r"-?\d+(\.\d+)?", round_number, wkt.strip().upper()) + for key, canonical_list in canonical_values.items(): + qlever_list = list(qlever_values[key]) + if len(canonical_list) != len(qlever_list): + return False + for numbers in canonical_list: + counterpart = next( + (q for q in qlever_list if close(numbers, q)), None + ) + if counterpart is None: + return False + qlever_list.remove(counterpart) + return True def entity_version(self, graph, entity_id): """ @@ -503,15 +537,22 @@ def compare_entity( f" edited since the endpoint's stream position" ) return "undecidable" - canonical = self.normalize_triples( + canonical, canonical_geo = self.normalize_triples( canonical_document, entity_id, exclude_counters ) - qlever = self.normalize_triples( + qlever, qlever_geo = self.normalize_triples( qlever_graph, entity_id, exclude_counters ) missing = canonical - qlever extra = qlever - canonical if not missing and not extra: + if not self.geo_values_match(canonical_geo, qlever_geo): + log.error( + f"{entity_id}: DIVERGENT at version {qlever_version}" + f" (geographic values differ by more than" + f" {GEO_TOLERANCE})" + ) + return "divergent" log.info( f"{entity_id}: exact match at version" f" {qlever_version} ({len(qlever):,} triples)" diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py index ca84781f..8f9a8349 100644 --- a/test/qlever/commands/test_check_sync_with_wikidata_methods.py +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -78,19 +78,27 @@ def test_canonical_term_non_numeric(self): f"<{WD}Q42>", ) - def test_canonical_wkt(self): - # Coordinates are rounded to 5 decimal places and the keyword is - # uppercased, matching the fixed-precision encoding of QLever. - self.assertEqual( - self.command.canonical_wkt( - "Point(-0.14544444444444 51.566527777778)" - ), - self.command.canonical_wkt("POINT(-0.145444 51.566528)"), + def test_geo_values_match(self): + # The observed encoding differences (up to 1e-5) must compare as + # equal, a difference above the tolerance must not. + key = "

" + self.assertTrue( + self.command.geo_values_match( + {key: [[5.483421, 50.642359]]}, + {key: [[5.48342, 50.642359]]}, + ) ) - self.assertEqual( - self.command.canonical_wkt("Point(5.483421 50.642359)"), - self.command.canonical_wkt("POINT(5.48342 50.642359)"), + self.assertTrue( + self.command.geo_values_match( + {key: [[50.81588]]}, {key: [[50.81587]]} + ) ) + self.assertFalse( + self.command.geo_values_match( + {key: [[50.81588]]}, {key: [[50.816]]} + ) + ) + self.assertFalse(self.command.geo_values_match({key: [[50.8]]}, {})) def test_normalize_triples_exclusions(self): graph = Graph() @@ -121,24 +129,33 @@ def test_normalize_triples_exclusions(self): URIRef("http://www.wikidata.org/value/def"), ) ) - with_counters = self.command.normalize_triples(graph, "Q42", False) - without_counters = self.command.normalize_triples(graph, "Q42", True) + with_counters, _ = self.command.normalize_triples(graph, "Q42", False) + without_counters, _ = self.command.normalize_triples( + graph, "Q42", True + ) # The reference type and the normalized-quantity link are always # excluded; the counter only with `exclude_counters`. self.assertEqual(len(with_counters), 2) self.assertEqual(len(without_counters), 1) def test_normalize_triples_geo(self): - # The two lexical forms of the same coordinate (canonical vs. the - # export of the fixed-precision encoding) must compare equal. + # Geographic values enter the set comparison only via a placeholder + # (so the presence of the triple is still compared exactly), and + # the values are collected for the tolerance comparison. canonical, qlever = Graph(), Graph() value = URIRef("http://www.wikidata.org/value/abc") latitude = URIRef(f"{WIKIBASE}geoLatitude") canonical.add((value, latitude, literal("51.566527777778", "double"))) qlever.add((value, latitude, literal("51.56652777778", "decimal"))) - self.assertEqual( - self.command.normalize_triples(canonical, "Q42", False), - self.command.normalize_triples(qlever, "Q42", False), + canonical_lines, canonical_geo = self.command.normalize_triples( + canonical, "Q42", False + ) + qlever_lines, qlever_geo = self.command.normalize_triples( + qlever, "Q42", False + ) + self.assertEqual(canonical_lines, qlever_lines) + self.assertTrue( + self.command.geo_values_match(canonical_geo, qlever_geo) ) def test_canonical_graph_without_munging(self): From 0e1cf6cd4e9fc9e877a0cb31a8ef2e6869de1997 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Wed, 29 Jul 2026 10:23:22 +0200 Subject: [PATCH 07/16] Handle ancient dates and recently merged entities Two findings from the third 1000-entity run: 1. QLever's export omits the timezone designator for dates with years outside [-9999, 9999] (`-11700-01-01T00:00:00` instead of `-11700-01-01T00:00:00Z`), so dates are now compared without it (all times in Wikidata are UTC, so it carries no information here). 2. A recently merged entity is served by `Special:EntityData` as a redirect stub without an HTTP redirect. Detect this at download time and report the entity as a redirect. The stub must in particular not go into the munged batch, because its dangling document node makes `munge.sh` graft its version onto the NEXT entity of the batch, which then reports a spurious version mismatch. As a second line of defense, an `owl:sameAs` on either side is also detected at comparison time. --- .../commands/check_sync_with_wikidata.py | 43 +++++++++++++++++++ .../test_check_sync_with_wikidata_methods.py | 20 +++++++++ 2 files changed, 63 insertions(+) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 03a82427..213d0b40 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -62,6 +62,11 @@ WKT_DATATYPE = "http://www.opengis.net/ont/geosparql#wktLiteral" GEO_TOLERANCE = 2e-5 +# The redirect marker written by a Wikidata merge. An entity can become a +# redirect DURING the check (observed live); it is then reported as a +# redirect, like the redirects that are already detected at download time. +OWL_SAMEAS = "http://www.w3.org/2002/07/owl#sameAs" + class CheckSyncWithWikidataCommand(QleverCommand): """ @@ -407,6 +412,17 @@ def canonical_term(self, term): slightly differently than IEEE string parsing, with differences observed in the 12th significant digit). """ + # QLever's export omits the timezone designator for dates with + # years outside [-9999, 9999] ("-11700-01-01T00:00:00" instead of + # "-11700-01-01T00:00:00Z"), so compare dates without it (all times + # in Wikidata are UTC, so it carries no information here). + if ( + isinstance(term, Literal) + and term.datatype is not None + and str(term.datatype) + == "http://www.w3.org/2001/XMLSchema#dateTime" + ): + return f'"{str(term).rstrip("Z")}"^^DATE' numeric_datatypes = { "http://www.w3.org/2001/XMLSchema#decimal", "http://www.w3.org/2001/XMLSchema#integer", @@ -521,6 +537,15 @@ def compare_entity( Compare the two documents of the given entity. Returns one of `match`, `divergent`, `undecidable`, or `error`. """ + for graph in (qlever_graph, canonical_document): + for target in graph.objects( + URIRef(f"{WD}{entity_id}"), URIRef(OWL_SAMEAS) + ): + log.info( + f"{entity_id}: redirect to" + f" {str(target).rsplit('/', 1)[-1]}, skipped" + ) + return "redirect" canonical_version = self.entity_version(canonical_document, entity_id) qlever_version = self.entity_version(qlever_graph, entity_id) if canonical_version is None or qlever_version is None: @@ -581,6 +606,24 @@ def check_batch(self, batch, endpoint, munge_script, keep_dir): for entity_id in batch: try: ttl_bytes, redirected_to = self.fetch_canonical(entity_id) + if redirected_to is None and b"sameAs" in ttl_bytes: + # A recently merged entity is served as a redirect stub + # (no HTTP redirect yet). It must not go into the munged + # batch: its dangling document node makes `munge.sh` + # graft its version onto the NEXT entity of the batch. + stub = Graph() + stub.parse(data=ttl_bytes, format="turtle") + target = next( + iter( + stub.objects( + URIRef(f"{WD}{entity_id}"), + URIRef(OWL_SAMEAS), + ) + ), + None, + ) + if target is not None: + redirected_to = str(target).rsplit("/", 1)[-1] if redirected_to is not None: log.info( f"{entity_id}: redirect to {redirected_to}, skipped" diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py index 8f9a8349..85befc2d 100644 --- a/test/qlever/commands/test_check_sync_with_wikidata_methods.py +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -68,6 +68,26 @@ def test_canonical_term_numeric(self): '"123456789012345678"^^NUM', ) + def test_canonical_term_datetime(self): + # The export omits the timezone designator for years outside + # [-9999, 9999], so dates are compared without it. + self.assertEqual( + self.command.canonical_term( + literal("-11700-01-01T00:00:00Z", "dateTime") + ), + self.command.canonical_term( + literal("-11700-01-01T00:00:00", "dateTime") + ), + ) + self.assertNotEqual( + self.command.canonical_term( + literal("2020-01-01T00:00:00Z", "dateTime") + ), + self.command.canonical_term( + literal("2021-01-01T00:00:00Z", "dateTime") + ), + ) + def test_canonical_term_non_numeric(self): self.assertEqual( self.command.canonical_term(Literal("hello", lang="en")), From 8aaba568cfc12b8f410bb5d4cf065c68838c4c98 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Wed, 29 Jul 2026 12:08:02 +0200 Subject: [PATCH 08/16] Retry the canonical download once on a transient server error A 1000-entity run hit a single transient HTTP 500 from wikidata.org on one download, which was counted as an error for that entity. Retry HTTP 5xx once (after a short pause); client errors are not retried. --- .../commands/check_sync_with_wikidata.py | 18 +++++++--- .../test_check_sync_with_wikidata_methods.py | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 213d0b40..56dcef6b 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -9,6 +9,7 @@ import subprocess import tempfile import time +import urllib.error import urllib.parse import urllib.request from decimal import Decimal, InvalidOperation @@ -205,7 +206,8 @@ def fetch_canonical(self, entity_id): """ Download the canonical TTL for the entity from `Special:EntityData`. Returns `(ttl_bytes, redirected_to)`, where `redirected_to` is not - `None` if the entity is a redirect. + `None` if the entity is a redirect. A transient server error + (HTTP 5xx) is retried once. """ url = ( "https://www.wikidata.org/wiki/Special:EntityData/" @@ -214,9 +216,17 @@ def fetch_canonical(self, entity_id): request = urllib.request.Request( url, headers={"User-Agent": USER_AGENT} ) - with urllib.request.urlopen(request, timeout=60) as response: - body = response.read() - final_url = response.url + for attempt in (1, 2): + try: + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read() + final_url = response.url + break + except urllib.error.HTTPError as e: + if attempt == 1 and e.code >= 500: + time.sleep(5) + continue + raise match = re.search(r"EntityData/(Q\d+)", final_url) redirected_to = None if match and match.group(1) != entity_id: diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py index 85befc2d..c2ac7c04 100644 --- a/test/qlever/commands/test_check_sync_with_wikidata_methods.py +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -1,4 +1,7 @@ +import io import unittest +import urllib.error +from unittest import mock from rdflib import Graph, Literal, URIRef @@ -68,6 +71,36 @@ def test_canonical_term_numeric(self): '"123456789012345678"^^NUM', ) + def test_fetch_canonical_retries_server_errors(self): + # A transient HTTP 5xx on the download is retried once. + error = urllib.error.HTTPError( + "url", 500, "Internal Server Error", {}, io.BytesIO() + ) + response = mock.MagicMock() + response.__enter__.return_value.read.return_value = b"ttl" + response.__enter__.return_value.url = ( + "https://www.wikidata.org/wiki/Special:EntityData/Q42.ttl" + ) + with ( + mock.patch( + "urllib.request.urlopen", side_effect=[error, response] + ), + mock.patch("time.sleep"), + ): + ttl_bytes, redirected_to = self.command.fetch_canonical("Q42") + self.assertEqual(ttl_bytes, b"ttl") + self.assertIsNone(redirected_to) + # A client error (4xx) is not retried. + client_error = urllib.error.HTTPError( + "url", 404, "Not Found", {}, io.BytesIO() + ) + with ( + mock.patch("urllib.request.urlopen", side_effect=[client_error]), + mock.patch("time.sleep"), + ): + with self.assertRaises(urllib.error.HTTPError): + self.command.fetch_canonical("Q42") + def test_canonical_term_datetime(self): # The export omits the timezone designator for years outside # [-9999, 9999], so dates are compared without it. From d5a3018a374cea7aa1d98cedc3e508102fd0c64c Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Fri, 14 Aug 2026 20:26:14 +0200 Subject: [PATCH 09/16] Add option `--set-runtime-parameters` to `qlever start` Companion to ad-freiburg/qlever#3132, which added the repeatable option `--set-runtime-parameter name=value` to `qlever-server` for setting any runtime parameter at startup. With this change, `qlever start` accepts the option in the plural form `--set-runtime-parameters` (as a space-separated list of `name=value` assignments) and adds it to the `qlever-server` command line with one `--set-runtime-parameter` per assignment. It is also settable via `SET_RUNTIME_PARAMETERS` in the `[server]` section of the `Qleverfile`. When the option is not given, the command line is unchanged, so that older `qlever-server` binaries without the option keep working. NOTE: This is more robust than applying the parameters via `qlever settings` after the start. With the new option, the parameters are already in effect when the server answers its first request, and they are set again automatically on every restart. They can still be changed while the server is running. --- src/qlever/commands/start.py | 9 +++++++++ src/qlever/qleverfile.py | 11 +++++++++++ test/qlever/commands/test_start_execute.py | 7 +++++++ test/qlever/commands/test_start_other_methods.py | 1 + 4 files changed, 28 insertions(+) diff --git a/src/qlever/commands/start.py b/src/qlever/commands/start.py index 6f858270..d61d68cb 100644 --- a/src/qlever/commands/start.py +++ b/src/qlever/commands/start.py @@ -49,6 +49,14 @@ def construct_command(args) -> str: f" --rebuild-keep-previous-index-dirs" f" {args.rebuild_keep_previous_index_dirs}" ) + # One `--set-runtime-parameter` per assignment (the option is not + # multitoken). + set_runtime_parameters = vars(args).get("set_runtime_parameters") + if set_runtime_parameters: + start_cmd += "".join( + f" --set-runtime-parameter {shlex.quote(assignment)}" + for assignment in set_runtime_parameters + ) if args.only_pso_and_pos_permutations: start_cmd += " --only-pso-and-pos-permutations" if args.use_patterns == "no": @@ -177,6 +185,7 @@ def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: "persist_updates", "rebuild_index_strategy", "rebuild_keep_previous_index_dirs", + "set_runtime_parameters", "only_pso_and_pos_permutations", "use_patterns", "use_text_index", diff --git a/src/qlever/qleverfile.py b/src/qlever/qleverfile.py index c254e77c..17d75c0e 100644 --- a/src/qlever/qleverfile.py +++ b/src/qlever/qleverfile.py @@ -404,6 +404,17 @@ def arg(*args, **kwargs): "most-recent-only (keep only the most recently created), " "original-and-most-recent (keep both)", ) + server_args["set_runtime_parameters"] = arg( + "--set-runtime-parameters", + nargs="+", + default=None, + metavar="NAME=VALUE", + help="Space-separated list of runtime parameters to set at " + "server startup, each in the form `name=value` (for the list of " + "runtime parameters and their default values, run " + "`qlever-server --set-runtime-parameter help`; they can also be " + "changed while the server is running, via `qlever settings`)", + ) server_args["only_pso_and_pos_permutations"] = arg( "--only-pso-and-pos-permutations", action="store_true", diff --git a/test/qlever/commands/test_start_execute.py b/test/qlever/commands/test_start_execute.py index d0f0d205..414ed4a7 100644 --- a/test/qlever/commands/test_start_execute.py +++ b/test/qlever/commands/test_start_execute.py @@ -22,6 +22,10 @@ def test_construct_command_with_if(): args.persist_updates = False args.rebuild_index_strategy = "automatic:10000:1000000:0.1" args.rebuild_keep_previous_index_dirs = "most-recent-only" + args.set_runtime_parameters = [ + "default-query-timeout=300s", + "rebuild-max-concurrent-permutation-pairs=1", + ] args.access_token = True args.only_pso_and_pos_permutations = True args.use_patterns = "no" @@ -46,6 +50,8 @@ def test_construct_command_with_if(): f" -a {args.access_token}" " --rebuild-index-strategy automatic:10000:1000000:0.1" " --rebuild-keep-previous-index-dirs most-recent-only" + " --set-runtime-parameter default-query-timeout=300s" + " --set-runtime-parameter rebuild-max-concurrent-permutation-pairs=1" " --only-pso-and-pos-permutations" " --no-patterns" " -t" @@ -72,6 +78,7 @@ def test_construct_command_without_if(): args.persist_updates = False args.rebuild_index_strategy = "manual" args.rebuild_keep_previous_index_dirs = "original-and-most-recent" + args.set_runtime_parameters = None args.access_token = False args.only_pso_and_pos_permutations = False args.use_patterns = True diff --git a/test/qlever/commands/test_start_other_methods.py b/test/qlever/commands/test_start_other_methods.py index 437b249f..f85be3b7 100644 --- a/test/qlever/commands/test_start_other_methods.py +++ b/test/qlever/commands/test_start_other_methods.py @@ -42,6 +42,7 @@ def test_relevant_qleverfile_arguments(self): "persist_updates", "rebuild_index_strategy", "rebuild_keep_previous_index_dirs", + "set_runtime_parameters", "only_pso_and_pos_permutations", "use_patterns", "use_text_index", From 28e3c16806b1ae7205ea7f881d526e2b63480835 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Sun, 16 Aug 2026 22:56:40 +0200 Subject: [PATCH 10/16] Show a progress bar (like `qlever update-wikidata`) The bar advances once per downloaded entity, which dominates the running time. The per-entity outcome lines keep printing above the bar via tqdm_logging_redirect, and the bar disappears when the check is done. --- .../commands/check_sync_with_wikidata.py | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 56dcef6b..6bbfdd4c 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -3,6 +3,7 @@ import glob import gzip import json +import logging import random import re import shutil @@ -16,6 +17,7 @@ from pathlib import Path from rdflib import Graph, Literal, URIRef +from tqdm.contrib.logging import tqdm_logging_redirect from qlever.command import QleverCommand from qlever.log import log @@ -604,12 +606,14 @@ def compare_entity( log.error(f" extra: {line}") return "divergent" - def check_batch(self, batch, endpoint, munge_script, keep_dir): + def check_batch(self, batch, endpoint, munge_script, keep_dir, pbar=None): """ Check a batch of entities: download the canonical data and query the endpoint pairwise (so that the version gate has the best chance), then munge the whole batch in ONE run of `munge.sh`, and compare - entity by entity. Returns a dict from entity ID to outcome. + entity by entity. Returns a dict from entity ID to outcome. The + progress bar `pbar` is advanced once per downloaded entity (the + download loop dominates the running time). """ outcomes = {} snapshots = [] @@ -647,6 +651,8 @@ def check_batch(self, batch, endpoint, munge_script, keep_dir): except Exception as e: log.warning(f"{entity_id}: check failed ({e})") outcomes[entity_id] = "error" + if pbar is not None: + pbar.update(1) time.sleep(1) batch_graph = None if munge_script is not None and snapshots: @@ -739,10 +745,19 @@ def batches(entity_ids): yield entity_ids[i : i + args.batch_size] outcomes = {} - for batch in batches(entities): - outcomes.update( - self.check_batch(batch, endpoint, munge_script, keep_dir) - ) + with tqdm_logging_redirect( + loggers=[logging.getLogger("qlever")], + desc="Entities", + total=len(entities), + leave=False, + bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt}{postfix}", + ) as pbar: + for batch in batches(entities): + outcomes.update( + self.check_batch( + batch, endpoint, munge_script, keep_dir, pbar + ) + ) retry = [e for e, o in outcomes.items() if o == "undecidable"] if retry: log.info( @@ -750,10 +765,19 @@ def batches(entity_ids): f" during the check ..." ) time.sleep(5) - for batch in batches(retry): - outcomes.update( - self.check_batch(batch, endpoint, munge_script, keep_dir) - ) + with tqdm_logging_redirect( + loggers=[logging.getLogger("qlever")], + desc="Entities", + total=len(retry), + leave=False, + bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt}{postfix}", + ) as pbar: + for batch in batches(retry): + outcomes.update( + self.check_batch( + batch, endpoint, munge_script, keep_dir, pbar + ) + ) counts = { status: sum(1 for o in outcomes.values() if o == status) From 36e8b606b0bd0401112a3585b99f7b2c82a06ef4 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Sun, 16 Aug 2026 23:41:47 +0200 Subject: [PATCH 11/16] Accept a `--sparql-endpoint` without scheme (prepend `http://`) Passing `--sparql-endpoint mino:7001` failed every entity with "unknown url type: mino". --- src/qlever/commands/check_sync_with_wikidata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 6bbfdd4c..ebab5756 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -695,8 +695,10 @@ def execute(self, args) -> bool: endpoint = ( args.sparql_endpoint if args.sparql_endpoint - else f"http://{args.host_name}:{args.port}" + else f"{args.host_name}:{args.port}" ) + if "://" not in endpoint: + endpoint = f"http://{endpoint}" munge_scripts = sorted(glob.glob("service-*/munge.sh")) if args.munge == "yes" and not munge_scripts: log.error( From 16b50f348861ed91c75815d7d92b1f2e651dd10c Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Sun, 16 Aug 2026 23:56:15 +0200 Subject: [PATCH 12/16] Show the last-updated date per entity and align the log lines Each per-entity line now shows `last updated ` after the version, and the entity ID is right-padded to the longest sampled ID so that the lines of a run are aligned. --- .../commands/check_sync_with_wikidata.py | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index ebab5756..1d615e02 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -77,7 +77,8 @@ class CheckSyncWithWikidataCommand(QleverCommand): """ def __init__(self): - pass + # Width for `qid`, set in `execute` once the entities are known. + self.qid_width = 0 def description(self) -> str: return ( @@ -542,6 +543,25 @@ def entity_version(self, graph, entity_id): return str(o) return None + def entity_date_modified(self, graph, entity_id): + """ + Return the `schema:dateModified` of the given entity in the given + graph. + """ + for o in graph.objects( + URIRef(f"{WD}{entity_id}"), URIRef(f"{SCHEMA}dateModified") + ): + return str(o) + return None + + def qid(self, entity_id): + """ + Return `:`, right-padded so that the log lines of the + different entities of a run are aligned (the padding width is set + in `execute` to the longest sampled entity ID). + """ + return f"{entity_id + ':':<{self.qid_width + 1}}" + def compare_entity( self, entity_id, qlever_graph, canonical_document, exclude_counters ): @@ -554,7 +574,7 @@ def compare_entity( URIRef(f"{WD}{entity_id}"), URIRef(OWL_SAMEAS) ): log.info( - f"{entity_id}: redirect to" + f"{self.qid(entity_id)} redirect to" f" {str(target).rsplit('/', 1)[-1]}, skipped" ) return "redirect" @@ -562,14 +582,17 @@ def compare_entity( qlever_version = self.entity_version(qlever_graph, entity_id) if canonical_version is None or qlever_version is None: log.warning( - f"{entity_id}: could not determine version" + f"{self.qid(entity_id)} could not determine version" f" (canonical: {canonical_version}," f" endpoint: {qlever_version})" ) return "error" + last_updated = self.entity_date_modified( + canonical_document, entity_id + ) or self.entity_date_modified(qlever_graph, entity_id) if canonical_version != qlever_version: log.info( - f"{entity_id}: version mismatch (canonical:" + f"{self.qid(entity_id)} version mismatch (canonical:" f" {canonical_version}, endpoint: {qlever_version})," f" edited since the endpoint's stream position" ) @@ -585,18 +608,21 @@ def compare_entity( if not missing and not extra: if not self.geo_values_match(canonical_geo, qlever_geo): log.error( - f"{entity_id}: DIVERGENT at version {qlever_version}" + f"{self.qid(entity_id)} DIVERGENT at version" + f" {qlever_version}, last updated {last_updated}" f" (geographic values differ by more than" f" {GEO_TOLERANCE})" ) return "divergent" log.info( - f"{entity_id}: exact match at version" - f" {qlever_version} ({len(qlever):,} triples)" + f"{self.qid(entity_id)} exact match at version" + f" {qlever_version}, last updated {last_updated}" + f" ({len(qlever):,} triples)" ) return "match" log.error( - f"{entity_id}: DIVERGENT at version {qlever_version}" + f"{self.qid(entity_id)} DIVERGENT at version {qlever_version}," + f" last updated {last_updated}" f" ({len(missing)} triples missing on the endpoint," f" {len(extra)} extra)" ) @@ -640,7 +666,8 @@ def check_batch(self, batch, endpoint, munge_script, keep_dir, pbar=None): redirected_to = str(target).rsplit("/", 1)[-1] if redirected_to is not None: log.info( - f"{entity_id}: redirect to {redirected_to}, skipped" + f"{self.qid(entity_id)} redirect to" + f" {redirected_to}, skipped" ) outcomes[entity_id] = "redirect" else: @@ -649,7 +676,7 @@ def check_batch(self, batch, endpoint, munge_script, keep_dir, pbar=None): ) snapshots.append((entity_id, ttl_bytes, qlever_graph)) except Exception as e: - log.warning(f"{entity_id}: check failed ({e})") + log.warning(f"{self.qid(entity_id)} check failed ({e})") outcomes[entity_id] = "error" if pbar is not None: pbar.update(1) @@ -687,7 +714,7 @@ def check_batch(self, batch, endpoint, munge_script, keep_dir, pbar=None): exclude_counters, ) except Exception as e: - log.warning(f"{entity_id}: check failed ({e})") + log.warning(f"{self.qid(entity_id)} check failed ({e})") outcomes[entity_id] = "error" return outcomes @@ -737,6 +764,7 @@ def execute(self, args) -> bool: args.num_entities, args.recent_fraction, args.seed ) log.info(f"Entities: {', '.join(entities)}") + self.qid_width = max(len(e) for e in entities) if args.batch_size < 1: log.error("`--batch-size` must be at least 1") From aa34dbbd0611d4e73a1cfd31210f36620c730f31 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Mon, 17 Aug 2026 00:31:05 +0200 Subject: [PATCH 13/16] Left-pad the triple count so the numbers are aligned Width 7 covers counts up to 999,999; larger counts (rare) simply use more space. --- src/qlever/commands/check_sync_with_wikidata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 1d615e02..f852bc57 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -617,7 +617,7 @@ def compare_entity( log.info( f"{self.qid(entity_id)} exact match at version" f" {qlever_version}, last updated {last_updated}" - f" ({len(qlever):,} triples)" + f" ({len(qlever):>7,} triples)" ) return "match" log.error( From 0cfafcb771b34baa2aff9c8b3387320574d44f9d Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Mon, 17 Aug 2026 00:40:18 +0200 Subject: [PATCH 14/16] Interleave the recently edited and the uniformly drawn entities Previously all recently edited entities were checked first; a seeded shuffle mixes both kinds, which makes the intermediate results more interesting to watch. --- src/qlever/commands/check_sync_with_wikidata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index f852bc57..eb2964d8 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -408,6 +408,9 @@ def sample_entities(self, num_entities, recent_fraction, seed): and r["title"] not in entities ] time.sleep(1) + # Mix the recently edited and the uniformly drawn entities, so that + # the intermediate results are a mix of both. + rng.shuffle(entities) return entities # Comparison. From 38793236d705698309d40bd4582c3c24f65f2edc Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Tue, 18 Aug 2026 02:38:41 +0200 Subject: [PATCH 15/16] Also check lexemes in `check-sync-with-wikidata` Lexemes take their own path into the index (unmunged lexemes dump, plus the munged flavor of the update stream) and went untested, which let the missing-lexemes defect of the 2026-08-10 index (qlever#3252) go unnoticed. The sample now draws a configurable fraction (default 20%, option `--lexeme-fraction`) from the lexeme namespace. The compared document of a lexeme includes its forms and senses with their statements. Lexemes are always compared without munging, and the flavor heterogeneities are excluded on both sides: the (stale) `Special:EntityData` document node, which still serves as the version gate for never-updated lexemes, the entity-level version and modification date, and the `rdfs:label` and `wikibase:Lexeme/Form/ Sense/Statement` type triples that the munged flavor drops. --- .../commands/check_sync_with_wikidata.py | 221 ++++++++++++++---- .../test_check_sync_with_wikidata_methods.py | 89 +++++++ 2 files changed, 261 insertions(+), 49 deletions(-) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index eb2964d8..27146e68 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -31,6 +31,16 @@ STATEMENT = "http://www.wikidata.org/entity/statement/" REFERENCE = "http://www.wikidata.org/reference/" VALUE = "http://www.wikidata.org/value/" +ONTOLEX = "http://www.w3.org/ns/lemon/ontolex#" + +# The IRI prefix of the `Special:EntityData` document node. For items, the +# munging moves its `schema:version` and `schema:dateModified` onto the +# entity and drops the node. The lexemes dump is ingested UNMUNGED (see the +# Wikidata Qleverfile), so for lexemes the index contains this node, and it +# goes stale when the entity is updated via the stream (the updater knows +# nothing about it). It is therefore excluded from the comparison and used +# only as a fallback for the version gate. +DOCUMENT = "https://www.wikidata.org/wiki/Special:EntityData/" # Entity-level counter triples that are contained in the full dump (and hence # in the index), but NOT in the output of `Special:EntityData`. They are @@ -50,6 +60,23 @@ EXCLUDED_TYPE_OBJECTS = {f"{WIKIBASE}Reference"} EXCLUDED_PREDICATES = {f"{WIKIBASE}quantityNormalized"} +# A lexeme in the index is heterogeneous in a second way: loaded from the +# UNMUNGED lexemes dump, but updated via the munged flavor of the stream. +# The munging drops the `rdfs:label` triples of a lexeme document (they +# duplicate `wikibase:lemma`, `ontolex:representation`, and +# `skos:definition`) and the `rdf:type` triples below (they duplicate the +# `ontolex:` types, or carry no information in the case of +# `wikibase:Statement`). A stream-touched lexeme therefore lacks them for +# the touched parts and keeps them for the rest, so they are excluded from +# the comparison of a lexeme on both sides. +RDFS_LABEL = "http://www.w3.org/2000/01/rdf-schema#label" +LEXEME_EXCLUDED_TYPE_OBJECTS = { + f"{WIKIBASE}Lexeme", + f"{WIKIBASE}Form", + f"{WIKIBASE}Sense", + f"{WIKIBASE}Statement", +} + # Geographic coordinates are stored by QLever in a fixed-precision encoding # and exported in a normalized form; differences up to 1e-5 degrees have # been observed (5.483421 came back as 5.48342, and 50.81588 as 50.81587). @@ -117,6 +144,16 @@ def additional_arguments(self, subparser) -> None: " (they exercise the update path); the rest is drawn uniformly" " (default: 0.5)", ) + subparser.add_argument( + "--lexeme-fraction", + type=float, + default=0.2, + help="Fraction of the sample drawn from lexemes; they are" + " deliberately oversampled relative to their share of the" + " entities, because they take a separate path into the index" + " (unmunged dump) that would otherwise go untested" + " (default: 0.2)", + ) subparser.add_argument( "--seed", type=int, @@ -167,26 +204,34 @@ def qlever_entity_graph(self, endpoint, entity_id): Fetch the full document of the given entity from the QLever endpoint: the triples with the entity as subject, the statement nodes reachable from the entity (plus their references and values), and the sitelink - article blocks. + article blocks. For a lexeme, the document also includes its forms + and senses, with their own statements; they are reached via the + zero-or-one property path below, which for an item simply yields the + item itself. The `schema:about` query also fetches the + `Special:EntityData` document node of a lexeme, which is needed for + the version gate (see `entity_version`) but excluded from the + comparison (see `normalize_triples`). """ e = f"<{WD}{entity_id}>" + root = f"{e} (<{ONTOLEX}lexicalForm>|<{ONTOLEX}sense>)? ?root ." # NOTE: Statement IRIs of old statements contain the entity ID in - # lowercase. + # lowercase. The IRIs of the statements of a form or sense start + # with the ID of the lexeme, so the prefix below covers them too. st = ( f'FILTER(STRSTARTS(STR(?st), "{STATEMENT}{entity_id}-")' f' || STRSTARTS(STR(?st), "{STATEMENT}{entity_id.lower()}-"))' ) queries = [ - f"CONSTRUCT {{ {e} ?p ?o }} WHERE {{ {e} ?p ?o }}", + f"CONSTRUCT {{ ?root ?p ?o }} WHERE {{ {root} ?root ?p ?o }}", f"CONSTRUCT {{ ?st ?p2 ?o2 }} WHERE" - f" {{ {e} ?p1 ?st . {st} ?st ?p2 ?o2 }}", + f" {{ {root} ?root ?p1 ?st . {st} ?st ?p2 ?o2 }}", f"CONSTRUCT {{ ?x ?p3 ?o3 }} WHERE" - f" {{ {e} ?p1 ?st . {st} ?st ?p2 ?x ." + f" {{ {root} ?root ?p1 ?st . {st} ?st ?p2 ?x ." f' FILTER(STRSTARTS(STR(?x), "http://www.wikidata.org/reference/")' f' || STRSTARTS(STR(?x), "http://www.wikidata.org/value/"))' f" ?x ?p3 ?o3 }}", f"CONSTRUCT {{ ?x2 ?p5 ?o5 }} WHERE" - f" {{ {e} ?p1 ?st . {st} ?st ?p2 ?x ." + f" {{ {root} ?root ?p1 ?st . {st} ?st ?p2 ?x ." f' FILTER(STRSTARTS(STR(?x), "http://www.wikidata.org/reference/"))' f" ?x ?p3 ?x2 ." f' FILTER(STRSTARTS(STR(?x2), "http://www.wikidata.org/value/"))' @@ -230,7 +275,7 @@ def fetch_canonical(self, entity_id): time.sleep(5) continue raise - match = re.search(r"EntityData/(Q\d+)", final_url) + match = re.search(r"EntityData/([QL]\d+)", final_url) redirected_to = None if match and match.group(1) != entity_id: redirected_to = match.group(1) @@ -282,10 +327,19 @@ def extract_document(self, graph, entity_id): ) document = Graph() statements = set() - for p, o in graph.predicate_objects(entity): - document.add((entity, p, o)) - if isinstance(o, URIRef) and str(o).startswith(statement_prefixes): - statements.add(o) + # The document of a lexeme also includes its forms and senses, with + # their own statements (their statement IRIs start with the ID of + # the lexeme, so the prefixes above cover them). + roots = {entity} + for link in (f"{ONTOLEX}lexicalForm", f"{ONTOLEX}sense"): + roots |= set(graph.objects(entity, URIRef(link))) + for root in roots: + for p, o in graph.predicate_objects(root): + document.add((root, p, o)) + if isinstance(o, URIRef) and str(o).startswith( + statement_prefixes + ): + statements.add(o) references_and_values = set() for statement in statements: for p, o in graph.predicate_objects(statement): @@ -367,8 +421,14 @@ def mediawiki_api(self, params): with urllib.request.urlopen(request, timeout=60) as response: return json.load(response) - def sample_entities(self, num_entities, recent_fraction, seed): - rng = random.Random(seed) + def sample_namespace( + self, num_entities, recent_fraction, rng, namespace, title_pattern + ): + """ + Sample entity IDs from the given namespace (items live in namespace + 0 with titles like `Q42`, lexemes in namespace 146 with titles like + `Lexeme:L42`); the ID is the first group of `title_pattern`. + """ num_recent = round(num_entities * recent_fraction) num_uniform = num_entities - num_recent entities = [] @@ -377,16 +437,16 @@ def sample_entities(self, num_entities, recent_fraction, seed): { "action": "query", "list": "recentchanges", - "rcnamespace": "0", + "rcnamespace": namespace, "rctype": "edit", "rclimit": "500", } ) titles = sorted( set( - rc["title"] + match.group(1) for rc in result["query"]["recentchanges"] - if re.fullmatch(r"Q\d+", rc["title"]) + if (match := re.fullmatch(title_pattern, rc["title"])) ) ) entities += rng.sample(titles, min(num_recent, len(titles))) @@ -395,21 +455,34 @@ def sample_entities(self, num_entities, recent_fraction, seed): { "action": "query", "list": "random", - "rnnamespace": "0", + "rnnamespace": namespace, "rnlimit": str( min(20, num_recent + num_uniform - len(entities)) ), } ) entities += [ - r["title"] + match.group(1) for r in result["query"]["random"] - if re.fullmatch(r"Q\d+", r["title"]) - and r["title"] not in entities + if (match := re.fullmatch(title_pattern, r["title"])) + and match.group(1) not in entities ] time.sleep(1) - # Mix the recently edited and the uniformly drawn entities, so that - # the intermediate results are a mix of both. + return entities + + def sample_entities( + self, num_entities, recent_fraction, lexeme_fraction, seed + ): + rng = random.Random(seed) + num_lexemes = round(num_entities * lexeme_fraction) + entities = self.sample_namespace( + num_entities - num_lexemes, recent_fraction, rng, "0", r"(Q\d+)" + ) + entities += self.sample_namespace( + num_lexemes, recent_fraction, rng, "146", r"Lexeme:(L\d+)" + ) + # Mix the recently edited and the uniformly drawn entities (and the + # items and lexemes), so that the intermediate results are a mix. rng.shuffle(entities) return entities @@ -474,9 +547,34 @@ def normalize_triples(self, graph, entity_id, exclude_counters): others. """ entity = f"{WD}{entity_id}" + is_lexeme = entity_id.startswith("L") lines = set() geo_values = {} for s, p, o in graph: + # The `Special:EntityData` document node of a lexeme is in the + # index (unmunged dump), but goes stale on the first update via + # the stream, so it takes part only in the version gate. + if str(s).startswith(DOCUMENT): + continue + # A lexeme that was updated via the stream has the version and + # modification date grafted onto the entity (munged flavor of + # the stream), an untouched one does not; both are in sync, so + # this heterogeneity is excluded from the comparison. + if ( + is_lexeme + and str(s) == entity + and str(p) in (f"{SCHEMA}version", f"{SCHEMA}dateModified") + ): + continue + # See the comment at `LEXEME_EXCLUDED_TYPE_OBJECTS`. + if is_lexeme and ( + str(p) == RDFS_LABEL + or ( + str(p) == RDF_TYPE + and str(o) in LEXEME_EXCLUDED_TYPE_OBJECTS + ) + ): + continue if ( exclude_counters and str(s) == entity @@ -539,22 +637,29 @@ def close(numbers_1, numbers_2): def entity_version(self, graph, entity_id): """ Return the `schema:version` of the given entity in the given graph. + For a lexeme that was loaded from the (unmunged) dump and never + updated since, the version sits on the `Special:EntityData` document + node instead of the entity; once the entity is updated via the + stream, the version on the entity is the current one and the one on + the document node is stale, which is why the entity is tried first. """ - for o in graph.objects( - URIRef(f"{WD}{entity_id}"), URIRef(f"{SCHEMA}version") - ): - return str(o) + for subject in (f"{WD}{entity_id}", f"{DOCUMENT}{entity_id}"): + for o in graph.objects( + URIRef(subject), URIRef(f"{SCHEMA}version") + ): + return str(o) return None def entity_date_modified(self, graph, entity_id): """ Return the `schema:dateModified` of the given entity in the given - graph. + graph (like `entity_version`, with the document node as fallback). """ - for o in graph.objects( - URIRef(f"{WD}{entity_id}"), URIRef(f"{SCHEMA}dateModified") - ): - return str(o) + for subject in (f"{WD}{entity_id}", f"{DOCUMENT}{entity_id}"): + for o in graph.objects( + URIRef(subject), URIRef(f"{SCHEMA}dateModified") + ): + return str(o) return None def qid(self, entity_id): @@ -684,22 +789,34 @@ def check_batch(self, batch, endpoint, munge_script, keep_dir, pbar=None): if pbar is not None: pbar.update(1) time.sleep(1) + # Lexemes are ingested into the index from the UNMUNGED lexemes dump + # (see the Wikidata Qleverfile), so their canonical data is compared + # without munging, whatever `--munge` says. batch_graph = None - if munge_script is not None and snapshots: - try: - batch_graph = self.munge( - b"".join(ttl for _, ttl, _ in snapshots), - munge_script, - keep_dir, - ) - except Exception as e: - log.warning(f"Munging the batch failed ({e})") - for entity_id, _, _ in snapshots: - outcomes[entity_id] = "error" - return outcomes + munge_failed = False + if munge_script is not None: + munge_batch = [ + (entity_id, ttl) + for entity_id, ttl, _ in snapshots + if not entity_id.startswith("L") + ] + if munge_batch: + try: + batch_graph = self.munge( + b"".join(ttl for _, ttl in munge_batch), + munge_script, + keep_dir, + ) + except Exception as e: + log.warning(f"Munging the batch failed ({e})") + munge_failed = True for entity_id, ttl_bytes, qlever_graph in snapshots: + is_lexeme = entity_id.startswith("L") + if munge_failed and not is_lexeme: + outcomes[entity_id] = "error" + continue try: - if batch_graph is not None: + if batch_graph is not None and not is_lexeme: canonical_document = self.extract_document( batch_graph, entity_id ) @@ -748,12 +865,14 @@ def execute(self, args) -> bool: return True keep_dir = Path.cwd() if args.keep_files else None - if not 0.0 <= args.recent_fraction <= 1.0: - log.error("`--recent-fraction` must be between 0.0 and 1.0") - return False + for name in ("recent_fraction", "lexeme_fraction"): + if not 0.0 <= getattr(args, name) <= 1.0: + option = name.replace("_", "-") + log.error(f"`--{option}` must be between 0.0 and 1.0") + return False if args.entities: entities = [e.strip() for e in args.entities.split(",")] - invalid = [e for e in entities if not re.fullmatch(r"Q\d+", e)] + invalid = [e for e in entities if not re.fullmatch(r"[QL]\d+", e)] if invalid: log.error(f"Invalid entity IDs: {invalid}") return False @@ -761,10 +880,14 @@ def execute(self, args) -> bool: log.info( f"Sampling {args.num_entities} entities" f" ({args.recent_fraction:.0%} recently edited," + f" {args.lexeme_fraction:.0%} lexemes," f" seed {args.seed}) ..." ) entities = self.sample_entities( - args.num_entities, args.recent_fraction, args.seed + args.num_entities, + args.recent_fraction, + args.lexeme_fraction, + args.seed, ) log.info(f"Entities: {', '.join(entities)}") self.qid_width = max(len(e) for e in entities) diff --git a/test/qlever/commands/test_check_sync_with_wikidata_methods.py b/test/qlever/commands/test_check_sync_with_wikidata_methods.py index c2ac7c04..124f27ef 100644 --- a/test/qlever/commands/test_check_sync_with_wikidata_methods.py +++ b/test/qlever/commands/test_check_sync_with_wikidata_methods.py @@ -275,6 +275,95 @@ def test_extract_document(self): ) self.assertEqual(len(document), 10) + def test_extract_document_lexeme(self): + # The document of a lexeme also includes its forms and senses, with + # their own statements (recognized by the same IRI prefix, which + # starts with the ID of the lexeme). + turtle = f""" + @prefix wd: <{WD}> . + @prefix wds: . + @prefix p: . + @prefix ontolex: . + wd:L42 ontolex:lexicalForm wd:L42-F1 ; + ontolex:sense wd:L42-S1 ; + p:P1 wds:L42-aaa . + wd:L42-F1 ontolex:representation "desks"@en ; + p:P2 wds:L42-F1-bbb . + wd:L42-S1 p:P3 wds:L42-S1-ccc . + wds:L42-aaa p:P4 "x" . + wds:L42-F1-bbb p:P5 "y" . + wds:L42-S1-ccc p:P6 "z" . + wd:L43 ontolex:lexicalForm wd:L43-F1 . + wd:L43-F1 p:P7 "other" . + """ + graph = Graph() + graph.parse(data=turtle, format="turtle") + document = self.command.extract_document(graph, "L42") + subjects = set(str(s) for s in document.subjects()) + self.assertEqual( + subjects, + { + f"{WD}L42", + f"{WD}L42-F1", + f"{WD}L42-S1", + "http://www.wikidata.org/entity/statement/L42-aaa", + "http://www.wikidata.org/entity/statement/L42-F1-bbb", + "http://www.wikidata.org/entity/statement/L42-S1-ccc", + }, + ) + self.assertEqual(len(document), 9) + + def test_entity_version_document_node_fallback(self): + # For a lexeme loaded from the (unmunged) dump, the version sits on + # the `Special:EntityData` document node; once the lexeme is updated + # via the stream, the entity carries the current version and the + # document node the stale one, so the entity is tried first. + schema_version = URIRef("http://schema.org/version") + document_node = URIRef( + "https://www.wikidata.org/wiki/Special:EntityData/L42" + ) + graph = Graph() + graph.add((document_node, schema_version, literal("123", "integer"))) + self.assertEqual(self.command.entity_version(graph, "L42"), "123") + graph.add( + (URIRef(f"{WD}L42"), schema_version, literal("456", "integer")) + ) + self.assertEqual(self.command.entity_version(graph, "L42"), "456") + + def test_normalize_triples_lexeme_exclusions(self): + # The document node and the entity-level version and modification + # date of a lexeme are excluded from the comparison (they are + # heterogeneous between dump-loaded and stream-updated lexemes); + # for an item, the entity-level version is compared. + schema_version = URIRef("http://schema.org/version") + graph = Graph() + graph.add( + ( + URIRef("https://www.wikidata.org/wiki/Special:EntityData/L42"), + schema_version, + literal("123", "integer"), + ) + ) + graph.add( + (URIRef(f"{WD}L42"), schema_version, literal("456", "integer")) + ) + graph.add( + ( + URIRef(f"{WD}L42"), + URIRef(f"{WIKIBASE}lemma"), + Literal("desk", lang="en"), + ) + ) + lines, _ = self.command.normalize_triples(graph, "L42", True) + self.assertEqual(len(lines), 1) + self.assertIn("lemma", next(iter(lines))) + item_graph = Graph() + item_graph.add( + (URIRef(f"{WD}Q42"), schema_version, literal("456", "integer")) + ) + lines, _ = self.command.normalize_triples(item_graph, "Q42", True) + self.assertEqual(len(lines), 1) + if __name__ == "__main__": unittest.main() From d6d9457113fe53271f354c8845e4f8b30d6978e1 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Thu, 20 Aug 2026 10:44:38 +0200 Subject: [PATCH 16/16] Normalize rendered MathML literals in `check-sync-with-wikidata` Defining-formula literals (P2534) are MathML rendered by MediaWiki's Math extension, and extension updates change the rendered markup retroactively without an entity edit. Observed on Q96843171: live EntityData now emits "mathjax_ignore" in the class list, while the dump-era index has the old rendering, which produced a spurious divergence at an unchanged version. The class list of the "mwe-math-element" is now normalized on both sides before the comparison; Q96843171 reconciles to an exact match. --- .../commands/check_sync_with_wikidata.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/qlever/commands/check_sync_with_wikidata.py b/src/qlever/commands/check_sync_with_wikidata.py index 27146e68..94ddaa02 100644 --- a/src/qlever/commands/check_sync_with_wikidata.py +++ b/src/qlever/commands/check_sync_with_wikidata.py @@ -92,6 +92,15 @@ WKT_DATATYPE = "http://www.opengis.net/ont/geosparql#wktLiteral" GEO_TOLERANCE = 2e-5 +# Defining-formula literals (P2534 and friends) are MathML RENDERED by +# MediaWiki's Math extension, and extension updates change the rendered +# markup retroactively without an entity edit (observed 2026-08-20 on +# Q96843171: "mathjax_ignore" appeared in the class list of live +# EntityData, while the dump-era index has the old rendering). The class +# list of such literals is therefore normalized before the comparison. +MATHML_DATATYPE = "http://www.w3.org/1998/Math/MathML" +MATHML_CLASS_RE = re.compile(r'class="mwe-math-element[^"]*"') + # The redirect marker written by a Wikidata merge. An entity can become a # redirect DURING the check (observed live); it is then reported as a # redirect, like the redirects that are already detected at download time. @@ -512,6 +521,16 @@ def canonical_term(self, term): == "http://www.w3.org/2001/XMLSchema#dateTime" ): return f'"{str(term).rstrip("Z")}"^^DATE' + # See the comment at `MATHML_DATATYPE`. + if ( + isinstance(term, Literal) + and term.datatype is not None + and str(term.datatype) == MATHML_DATATYPE + ): + normalized = MATHML_CLASS_RE.sub( + 'class="mwe-math-element"', str(term) + ) + return f"{json.dumps(normalized)}^^MATHML" numeric_datatypes = { "http://www.w3.org/2001/XMLSchema#decimal", "http://www.w3.org/2001/XMLSchema#integer",