diff --git a/.gitignore b/.gitignore index 762a39c5..a42cc3c2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ __pycache__ build/ dist/ *.egg-info +*.memory-log.json *.swp diff --git a/README.md b/README.md index 866c3765..febaec02 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,15 @@ On other platforms simply install the `qlever` command-line [python package using `pipx`/`uv`](https://docs.qlever.dev/quickstart/#others). Note: QLever will be executed in a container which will come with a performance penalty. +# Optional SPARQL conformance integration + +The `qlever` package is independent of the SPARQL conformance test harness and +does not install it. To use the integrated +`sparql_conformance setup|test|analyze|visualize` workflow, install the +[`sparql-conformance`](https://github.com/SIRDNARch/sparql-conformance) +package separately. That package owns the `sparql_conformance` command and +loads qlever-control only when the integration is used. + # Use with your own dataset To use QLever with your own dataset, you need a `Qleverfile`, like in the diff --git a/pyproject.toml b/pyproject.toml index cbcdb85d..cd55f2c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=61.0", "wheel"] +requires = ["setuptools>=64.0", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -33,7 +33,13 @@ repository = "https://github.com/ad-freiburg/qlever.git" bugtracker = "https://github.com/ad-freiburg/qlever/issues" [project.scripts] +"qblazegraph" = "qlever.qlever_main:main" +"qgraphdb" = "qlever.qlever_main:main" +"qjena" = "qlever.qlever_main:main" "qlever" = "qlever.qlever_main:main" +"qmdb" = "qlever.qlever_main:main" +"qoxigraph" = "qlever.qlever_main:main" +"qvirtuoso" = "qlever.qlever_main:main" [tool.setuptools] package-data = { "qlever" = ["Qleverfiles/*", "evaluation/www/*"] } diff --git a/src/qblazegraph/Dockerfile b/src/qblazegraph/Dockerfile new file mode 100644 index 00000000..6711f7f2 --- /dev/null +++ b/src/qblazegraph/Dockerfile @@ -0,0 +1,23 @@ +# Use an official OpenJDK runtime as a parent image +FROM docker.io/library/openjdk:26-ea-21-jdk-slim + +RUN apt-get update && apt-get install -y wget gzip coreutils curl unzip + +WORKDIR /opt + +# Download and set up Blazegraph +RUN wget https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar + +# Set ownership to the user passed by UID and GID +ARG UID +ARG GID +RUN if [ "${UID:-}" != "" ] && [ "${GID:-}" != "" ]; then \ + chown -R ${UID}:${GID} /opt; \ + fi + +RUN chmod u+x blazegraph.jar + +# Expose Blazegraph port +# EXPOSE 9999 + +CMD ["bash"] diff --git a/src/qblazegraph/RWStore.conformance.properties b/src/qblazegraph/RWStore.conformance.properties new file mode 100644 index 00000000..34b57539 --- /dev/null +++ b/src/qblazegraph/RWStore.conformance.properties @@ -0,0 +1,33 @@ +# ===================== Basic Configuration ===================== +com.bigdata.journal.AbstractJournal.file=blazegraph.jnl +com.bigdata.journal.AbstractJournal.bufferMode=DiskRW +com.bigdata.journal.AbstractJournal.initialExtent=209715200 +com.bigdata.journal.AbstractJournal.maximumExtent=200000000000 +com.bigdata.rdf.store.AbstractTripleStore.textIndex=false +com.bigdata.rdf.store.AbstractTripleStore.justify=false +com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms +com.bigdata.rdf.store.AbstractTripleStore.quads=true +com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=false +com.bigdata.rdf.sail.truthMaintenance=false +com.bigdata.rdf.store.AbstractTripleStore.geoSpatial=false + +# ===================== DataLoader Performance ===================== +com.bigdata.rdf.store.DataLoader.bufferCapacity=1000000 +com.bigdata.rdf.store.DataLoader.queueCapacity=150 +com.bigdata.rdf.store.DataLoader.gzipBufferSize=1000000000 +com.bigdata.rdf.store.DataLoader.closure=None + +# ===================== BTree Tuning (example values; adjust based on your dataset) ===================== +# com.bigdata.namespace.kb.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=576 +# com.bigdata.namespace.kb.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=781 +# com.bigdata.namespace.kb.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=380 +# com.bigdata.namespace.kb.spo.OSP.com.bigdata.btree.BTree.branchingFactor=737 +# com.bigdata.namespace.kb.spo.POS.com.bigdata.btree.BTree.branchingFactor=917 +# com.bigdata.namespace.kb.spo.SPO.com.bigdata.btree.BTree.branchingFactor=731 + +# ===================== Optional Namespace Settings ===================== +com.bigdata.namespace.kb.spo.com.bigdata.btree.BTree.branchingFactor=1024 +com.bigdata.rdf.store.AbstractTripleStore.namespace=kb + +# ===================== RDF & Parser Configuration ===================== +com.bigdata.rdf.parser.RDFParserFactory.supportsGZip=true diff --git a/src/qblazegraph/RWStore.properties b/src/qblazegraph/RWStore.properties new file mode 100644 index 00000000..cf2339e1 --- /dev/null +++ b/src/qblazegraph/RWStore.properties @@ -0,0 +1,33 @@ +# ===================== Basic Configuration ===================== +com.bigdata.journal.AbstractJournal.file=blazegraph.jnl +com.bigdata.journal.AbstractJournal.bufferMode=DiskRW +com.bigdata.journal.AbstractJournal.initialExtent=209715200 +com.bigdata.journal.AbstractJournal.maximumExtent=200000000000 +com.bigdata.rdf.store.AbstractTripleStore.textIndex=false +com.bigdata.rdf.store.AbstractTripleStore.justify=false +com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms +com.bigdata.rdf.store.AbstractTripleStore.quads=false +com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=false +com.bigdata.rdf.sail.truthMaintenance=false +com.bigdata.rdf.store.AbstractTripleStore.geoSpatial=false + +# ===================== DataLoader Performance ===================== +com.bigdata.rdf.store.DataLoader.bufferCapacity=1000000 +com.bigdata.rdf.store.DataLoader.queueCapacity=150 +com.bigdata.rdf.store.DataLoader.gzipBufferSize=1000000000 +com.bigdata.rdf.store.DataLoader.closure=None + +# ===================== BTree Tuning (example values; adjust based on your dataset) ===================== +# com.bigdata.namespace.kb.lex.BLOBS.com.bigdata.btree.BTree.branchingFactor=576 +# com.bigdata.namespace.kb.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=781 +# com.bigdata.namespace.kb.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=380 +# com.bigdata.namespace.kb.spo.OSP.com.bigdata.btree.BTree.branchingFactor=737 +# com.bigdata.namespace.kb.spo.POS.com.bigdata.btree.BTree.branchingFactor=917 +# com.bigdata.namespace.kb.spo.SPO.com.bigdata.btree.BTree.branchingFactor=731 + +# ===================== Optional Namespace Settings ===================== +com.bigdata.namespace.kb.spo.com.bigdata.btree.BTree.branchingFactor=1024 +com.bigdata.rdf.store.AbstractTripleStore.namespace=kb + +# ===================== RDF & Parser Configuration ===================== +com.bigdata.rdf.parser.RDFParserFactory.supportsGZip=true \ No newline at end of file diff --git a/src/qblazegraph/__init__.py b/src/qblazegraph/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qblazegraph/commands/__init__.py b/src/qblazegraph/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qblazegraph/commands/benchmark_queries.py b/src/qblazegraph/commands/benchmark_queries.py new file mode 100644 index 00000000..4327d79f --- /dev/null +++ b/src/qblazegraph/commands/benchmark_queries.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from qlever.commands.benchmark_queries import ( + BenchmarkQueriesCommand as QleverBenchmarkQueriesCommand, +) + + +class BenchmarkQueriesCommand(QleverBenchmarkQueriesCommand): + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "server": ["host_name", "port", "timeout"], + "ui": ["ui_config"], + } + + def execute(self, args) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = ( + f"{args.host_name}:{args.port}/blazegraph/namespace/kb/sparql" + ) + return super().execute(args) diff --git a/src/qblazegraph/commands/extract_queries.py b/src/qblazegraph/commands/extract_queries.py new file mode 120000 index 00000000..5667cc52 --- /dev/null +++ b/src/qblazegraph/commands/extract_queries.py @@ -0,0 +1 @@ +../../qlever/commands/extract_queries.py \ No newline at end of file diff --git a/src/qblazegraph/commands/get_data.py b/src/qblazegraph/commands/get_data.py new file mode 120000 index 00000000..4900dbb8 --- /dev/null +++ b/src/qblazegraph/commands/get_data.py @@ -0,0 +1 @@ +../../qlever/commands/get_data.py \ No newline at end of file diff --git a/src/qblazegraph/commands/index.py b/src/qblazegraph/commands/index.py new file mode 100644 index 00000000..e7a7d6af --- /dev/null +++ b/src/qblazegraph/commands/index.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from pathlib import Path + +import qlever.util as util +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log + + +class IndexCommand(QleverCommand): + def __init__(self): + self.script_name = "qblazegraph" + + def description(self) -> str: + return "Build the index for a given RDF dataset" + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name", "format"], + "index": ["input_files", "jvm_args", "extra_args"], + "runtime": ["system", "image", "index_container"], + } + + def additional_arguments(self, subparser): + subparser.add_argument( + "--blazegraph-jar", + type=str, + default="blazegraph.jar", + help=( + "Path to blazegraph.jar file (default: blazegraph.jar) " + "(this requires that you have Java installed and blazegraph.jar " + "downloaded on your machine)" + ), + ) + subparser.add_argument( + "--rebuild-image", + action="store_true", + default=False, + help="Rebuild the Docker image to get the latest updates", + ) + + @staticmethod + def wrap_cmd_in_container(args, cmd: str) -> str: + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand="run --rm", + image_name=args.image, + container_name=args.index_container, + volumes=[("$(pwd)", "/opt/index")], + working_directory="/opt/index", + ) + + def execute(self, args, called_from_conformance_test: bool = False) -> bool: + system = args.system + input_files = args.input_files + + jar_path = ( + args.blazegraph_jar + if args.system == "native" + else "/opt/blazegraph.jar" + ) + + index_cmd = ( + f"java {args.jvm_args} -cp {jar_path} com.bigdata.rdf.store.DataLoader " + f"{args.extra_args} RWStore.properties {input_files}" + ) + # Under a conformance test, redirect all output to the log file so the + # BlazeGraph banner and DataLoader output do not flood stdout; otherwise + # `tee` keeps showing it live. + if called_from_conformance_test: + index_cmd += f" > {args.name}.index-log.txt 2>&1" + else: + index_cmd += f" | tee {args.name}.index-log.txt" + + image_id = build_cmd = "" + if args.system == "native": + cmd_to_show = index_cmd + else: + index_cmd = self.wrap_cmd_in_container(args, index_cmd) + dockerfile_dir = Path(__file__).parent.parent + dockerfile_path = dockerfile_dir / "Dockerfile" + build_cmd = ( + f"{system} build -f {dockerfile_path} -t {args.image} --build-arg " + f"UID=$(id -u) --build-arg GID=$(id -g) {dockerfile_dir}" + ) + image_id = util.get_container_image_id(system, args.image) + cmd_to_show = ( + f"{build_cmd}\n\n{index_cmd}" + if not image_id or args.rebuild_image + else index_cmd + ) + + # Show the command line. + self.show(cmd_to_show, only_show=args.show) + if args.show: + return True + + # Check if all of the input files exist. + if not util.input_files_exist(input_files): + return False + + # When running natively, check if the binary exists and works. + if args.system == "native": + try: + util.run_command("java --help") + except Exception as e: + log.error(f"Java not found on the machine! - {e}") + log.info( + "Blazegraph needs Java to execute the blazegraph.jar file" + ) + return False + if not Path(args.blazegraph_jar).exists(): + jar_link = ( + "https://github.com/blazegraph/database/releases/download/" + "BLAZEGRAPH_2_1_6_RC/blazegraph.jar" + ) + log.error( + "Couldn't find the blazegraph.jar in specified path: " + f"{Path(args.blazegraph_jar).absolute()}\n" + ) + log.info( + "Are you sure you downloaded the blazegraph.jar file? " + f"blazegraph.jar can be downloaded from {jar_link}" + ) + return False + else: + if Containerize().is_running(args.system, args.index_container): + log.info( + f"{args.system} container {args.index_container} is still up, " + "which means that data loading is in progress. Please wait..." + ) + return False + # A previously interrupted run can leave a container with this name + # in a non-running (Created/Exited) state, which makes the + # `docker/podman run --name` below fail with exit code 125. Remove + # any such stale container first. + Containerize.stop_and_remove_container( + args.system, args.index_container + ) + + if not image_id or args.rebuild_image: + build_successful = util.build_image( + build_cmd, system, args.image + ) + if not build_successful: + return False + else: + log.info(f"{args.image} image present on the system\n") + + index_jnl = Path("blazegraph.jnl") + if index_jnl.exists(): + log.error( + "Blazegraph journal blazegraph.jnl found in current working " + "directory which shows presence of a previous index\n" + ) + log.info("Aborting the index operation...") + return False + + # Run the index command. + try: + util.run_command( + index_cmd, show_output=not called_from_conformance_test + ) + except Exception as e: + log.error(f"Building the index failed: {e}") + return False + + return True diff --git a/src/qblazegraph/commands/index_stats.py b/src/qblazegraph/commands/index_stats.py new file mode 100644 index 00000000..ad4bfae9 --- /dev/null +++ b/src/qblazegraph/commands/index_stats.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import re + +from qlever.commands.index_stats import ( + IndexStatsCommand as QleverIndexStatsCommand, +) +from qlever.log import log +from qlever.util import get_total_file_size, run_command + + +class IndexStatsCommand(QleverIndexStatsCommand): + """ + Class for executing the `index-stats` command. + """ + + def execute_time( + self, args, log_file_name: str + ) -> dict[str, tuple[float | None, str]]: + """ + Part of `execute` that returns the time used for each part of indexing + along with the unit. + """ + + # Read the content of `log_file_name` into a list of lines. + try: + log_text = run_command( + f"tail {log_file_name}", return_output=True + ) + except Exception as e: + log.error(f"Problem reading index log file {log_file_name}: {e}") + return {} + + stats = {} + # Pattern for the overall line in seconds + overall_pattern = re.compile(r"Total elapsed=(\d+)ms") + + for line in log_text.splitlines(): + label = raw_value = None + overall_match = overall_pattern.search(line) + if overall_match: + label = "TOTAL time" + raw_value = overall_match.group(1) + + if raw_value is None: + continue + + try: + value_s = float(raw_value) / 1000 + except (ValueError, TypeError): + continue + + time_unit = self.get_time_unit(args.time_unit, value_s) + unit_factor = self.get_time_unit_factor(time_unit) + + normalized_value = value_s / unit_factor + stats[label] = (normalized_value, time_unit) + + if overall_match: + break + + return stats + + def execute_space(self, args) -> dict[str, tuple[float, str]]: + """ + Part of `execute` that returns the space used by different types of + index along with the unit. + """ + index_size = get_total_file_size(["blazegraph.jnl"]) + + size_unit = self.get_size_unit(args.size_unit, index_size) + unit_factor = self.get_size_unit_factor(size_unit) + + index_size /= unit_factor + + return {"TOTAL size": (index_size, size_unit)} diff --git a/src/qblazegraph/commands/log.py b/src/qblazegraph/commands/log.py new file mode 120000 index 00000000..90ffabb4 --- /dev/null +++ b/src/qblazegraph/commands/log.py @@ -0,0 +1 @@ +../../qlever/commands/log.py \ No newline at end of file diff --git a/src/qblazegraph/commands/query.py b/src/qblazegraph/commands/query.py new file mode 100644 index 00000000..eeaf2323 --- /dev/null +++ b/src/qblazegraph/commands/query.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from qoxigraph.commands.query import QueryCommand as QoxigraphQueryCommand + + +class QueryCommand(QoxigraphQueryCommand): + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return {"data": ["name"], "server": ["port", "access_token"]} + + def execute(self, args) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = ( + f"localhost:{args.port}/blazegraph/namespace/kb/sparql" + ) + return super().execute(args) diff --git a/src/qblazegraph/commands/setup_config.py b/src/qblazegraph/commands/setup_config.py new file mode 100644 index 00000000..5b6348dd --- /dev/null +++ b/src/qblazegraph/commands/setup_config.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import shutil +from pathlib import Path + +from qlever.log import log +from qlever.util import add_memory_options, run_curl_command +from qoxigraph.commands.setup_config import ( + SetupConfigCommand as QoxigraphSetupConfigCommand, +) + + +class SetupConfigCommand(QoxigraphSetupConfigCommand): + """ + Should behave exactly the same as setup-config command in qoxigraph, + just with a different Docker image name + """ + + IMAGE = "adfreiburg/qblazegraph" + + @staticmethod + def construct_engine_specific_params(args) -> dict[str, dict[str, str]]: + index_memory = args.total_index_memory + server_memory = args.total_server_memory + return { + "index": { + "JVM_ARGS": f"-Xms{index_memory} -Xmx{index_memory}" + }, + "server": { + "JVM_ARGS": f"-Xms{server_memory} -Xmx{server_memory}", + "TIMEOUT": "60s", + "READ_ONLY": "yes", + }, + } + + def additional_arguments(self, subparser) -> None: + super().additional_arguments(subparser) + add_memory_options(subparser) + + def execute(self, args) -> bool: + qleverfile_successfully_created = super().execute(args) + if not qleverfile_successfully_created: + return False + + properties_file_path = ( + Path(__file__).parent.parent / "RWStore.properties" + ) + destination = Path("RWStore.properties") + try: + shutil.copy(properties_file_path, destination) + log.info("Copied RWStore.properties to current directory!") + except Exception as e: + file_url = ( + "https://github.com/ad-freiburg/qlever-control/tree/main/src/" + "qblazegraph/RWStore.properties" + ) + log.error( + "Couldn't copy RWStore.properties file to current working " + f"directory! Error: {e}\n" + ) + log.info(f"Download it manually from {file_url}") + return False + web_xml_url = ( + "https://raw.githubusercontent.com/blazegraph/database/refs/heads/master/" + "bigdata-war-html/src/main/webapp/WEB-INF/web.xml" + ) + try: + run_curl_command(url=web_xml_url, result_file="web.xml") + log.info( + "Successfully downloaded web.xml for specifying config parameters." + ) + return True + except Exception as e: + log.error(f"Failed to download web.xml file: {e}") + log.info(f"Download it manually from {web_xml_url}") + return False diff --git a/src/qblazegraph/commands/start.py b/src/qblazegraph/commands/start.py new file mode 100644 index 00000000..178f49de --- /dev/null +++ b/src/qblazegraph/commands/start.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import subprocess +import time +import xml.etree.ElementTree as ET +from pathlib import Path + +from qblazegraph.commands.stop import StopCommand +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log +from qlever.util import is_server_alive, run_command + + +class StartCommand(QleverCommand): + def __init__(self): + self.script_name = "qblazegraph" + + def description(self) -> str: + return ( + "Start the server for Blazegraph (requires that you have built an " + "index before)" + ) + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "server": [ + "host_name", + "port", + "jvm_args", + "read_only", + "timeout", + "extra_args", + ], + "runtime": ["system", "image", "server_container"], + } + + def additional_arguments(self, subparser): + subparser.add_argument( + "--run-in-foreground", + action="store_true", + default=False, + help=( + "Run the start command in the foreground " + "(default: run in the background)" + ), + ) + subparser.add_argument( + "--blazegraph-jar", + type=str, + default="blazegraph.jar", + help=( + "Path to blazegraph.jar file (default: blazegraph.jar) " + "(this requires that you have Java installed and blazegraph.jar " + "downloaded on your machine)" + ), + ) + + @staticmethod + def overwrite_web_xml( + xml_file_path: Path, timeout_ms: int, read_only: bool + ) -> None: + """ + Overwrite readOnly and queryTimeout parameters in web.xml + This method could be made more general by making new_value dict + itself an input parameter to the function. But for now, I could only + identify readOnly and queryTimeout as the 2 parameters that would + need updating and it is better to be explicit about it. + """ + new_values = { + "queryTimeout": str(timeout_ms), + "readOnly": str(read_only).lower(), + } + ns_uri = "http://java.sun.com/xml/ns/javaee" + namespace = {"ns": "http://java.sun.com/xml/ns/javaee"} + + # Register the default namespace to avoid ns0 prefixes + ET.register_namespace("", ns_uri) + + # Parse the XML and preserve comments + parser = ET.XMLParser(target=ET.TreeBuilder(insert_comments=True)) + tree = ET.parse(xml_file_path, parser=parser) + root = tree.getroot() + + # Find and update values + for context_param in root.findall("ns:context-param", namespace): + param_name = context_param.find("ns:param-name", namespace) + if param_name is not None and param_name.text in new_values: + param_value = context_param.find("ns:param-value", namespace) + if param_value is not None: + param_value.text = new_values[param_name.text] + tree.write(xml_file_path, encoding="UTF-8", xml_declaration=True) + log.info("Successfully updated web.xml.") + + @staticmethod + def wrap_cmd_in_container(args, cmd: str) -> str: + run_subcommand = "run --restart=unless-stopped" + if not args.run_in_foreground: + run_subcommand += " -d" + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand=run_subcommand, + image_name=args.image, + container_name=args.server_container, + volumes=[("$(pwd)", "/opt/index")], + working_directory="/opt/index", + ports=[(args.port, args.port)], + ) + + def execute(self, args, called_from_conformance_test: bool = False) -> bool: + jar_path = ( + args.blazegraph_jar + if args.system == "native" + else "/opt/blazegraph.jar" + ) + + xml_candidates = [Path("web.xml"), Path(f"{args.name}.web.xml")] + existing_files = [p for p in xml_candidates if p.is_file()] + if len(existing_files) > 1: + log.error( + "Expected exactly one of 'web.xml' or " + f"'{args.name}.web.xml' in the current directory." + ) + return False + web_xml_exists = len(existing_files) == 1 + web_xml_path = existing_files[0] if web_xml_exists else None + + if web_xml_exists: + log.info( + f"queryTimeout and readOnly parameters would be overwritten in {web_xml_path.name}" + ) + if web_xml_path.name == "web.xml": + log.info(f"web.xml would be renamed to {args.name}.web.xml") + log.info("") + start_cmd = ( + f"java -server {args.jvm_args} -Dbigdata.propertyFile=RWStore.properties" + f"{'' if not web_xml_exists else f' -Djetty.overrideWebXml={args.name}.web.xml'} " + f"-Djetty.port={args.port} {args.extra_args} -jar {jar_path}" + f" > {args.name}.server-log.txt 2>&1" + ) + + if args.system == "native": + if not args.run_in_foreground: + start_cmd = f"nohup {start_cmd} &" + else: + start_cmd = self.wrap_cmd_in_container(args, start_cmd) + + # Show the command line. + self.show(start_cmd, only_show=args.show) + if args.show: + return True + + # When running natively, check if the binary exists and works. + if args.system == "native": + try: + run_command("java --help") + except Exception as e: + log.error(f"Java not found on the machine! - {e}") + log.info( + "Blazegraph needs Java to execute the blazegraph.jar file" + ) + return False + if not Path(args.blazegraph_jar).exists(): + jar_link = ( + "https://github.com/blazegraph/database/releases/download/" + "BLAZEGRAPH_2_1_6_RC/blazegraph.jar" + ) + log.error( + "Couldn't find the blazegraph.jar in specified path: " + f"{Path(args.blazegraph_jar).absolute()}\n" + ) + log.info( + "Are you sure you downloaded the blazegraph.jar file? " + f"blazegraph.jar can be downloaded from {jar_link}" + ) + return False + + jnl_file = Path("blazegraph.jnl") + if not jnl_file.exists(): + log.info(f"No Blazegraph journal for {args.name} found! ") + log.info( + f"Did you call `{self.script_name} index`? If you did, check " + "if blazegraph.jnl is present in the current working directory" + ) + return False + + endpoint_url = f"http://{args.host_name}:{args.port}/blazegraph" + if is_server_alive(url=endpoint_url): + log.error(f"Blazegraph server already running on {endpoint_url}\n") + log.info( + f"To kill the existing server, use `{self.script_name} stop`" + ) + return False + + try: + timeout_ms = int(args.timeout[:-1]) * 1000 + except ValueError as e: + log.error(f"Invalid timeout value {args.timeout}. Error: {e}") + return False + + try: + if web_xml_exists: + read_only = True if args.read_only == "yes" else False + self.overwrite_web_xml(web_xml_path, timeout_ms, read_only) + if web_xml_path.name == "web.xml": + Path("web.xml").rename(f"{args.name}.web.xml") + log.info( + f"Successfully renamed web.xml to {args.name}.web.xml\n" + ) + except Exception as e: + log.error( + f"Overwriting web.xml with Qleverfile parameters failed: {e}" + ) + return False + + # Run the start command. + try: + process = run_command( + start_cmd, + use_popen=args.run_in_foreground, + ) + except Exception as e: + log.error(f"Starting the Jena server failed ({e})") + return False + + log_proc = None + if not called_from_conformance_test: + # Tail the server log until the server is ready (note that the + # `exec` is important to make sure that the tail process is killed + # and not just the bash process). + if args.run_in_foreground: + log.info( + "Follow the server logs as long as the server is" + " running (Ctrl-C stops the server)" + ) + else: + log.info( + "Follow the server logs until the server is ready" + " (Ctrl-C stops following the log, but NOT the server)" + ) + log.info("") + log_file = Path(f"{args.name}.server-log.txt") + while not log_file.exists(): + time.sleep(0.1) + log_cmd = f"exec tail -f {log_file}" + log_proc = subprocess.Popen(log_cmd, shell=True) + while not is_server_alive(endpoint_url): + time.sleep(1) + + if not called_from_conformance_test: + log.info( + f"Blazegraph server webapp for {args.name} will be available " + f"at http://{args.host_name}:{args.port} and the sparql " + f"endpoint for queries is {endpoint_url}/namespace/kb/sparql" + ) + + # Kill the log process + if log_proc is not None and not args.run_in_foreground: + log_proc.terminate() + + # With `--run-in-foreground`, wait until the server is stopped. + if args.run_in_foreground: + try: + process.wait() + except KeyboardInterrupt: + process.terminate() + if args.system in Containerize.supported_systems(): + args.cmdline_regex = StopCommand.DEFAULT_REGEX + StopCommand().execute(args) + if log_proc is not None: + log_proc.terminate() + + return True diff --git a/src/qblazegraph/commands/status.py b/src/qblazegraph/commands/status.py new file mode 100644 index 00000000..107b0481 --- /dev/null +++ b/src/qblazegraph/commands/status.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from qoxigraph.commands.status import StatusCommand as QoxigraphStatusCommand + + +class StatusCommand(QoxigraphStatusCommand): + DEFAULT_REGEX = "java\\s+-server.*blazegraph.jar" + + def description(self) -> str: + return ( + "Show Java processes with blazegraph.jar running on this machine" + ) diff --git a/src/qblazegraph/commands/stop.py b/src/qblazegraph/commands/stop.py new file mode 100644 index 00000000..052ace25 --- /dev/null +++ b/src/qblazegraph/commands/stop.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from qblazegraph.commands.status import StatusCommand +from qoxigraph.commands.stop import StopCommand as QoxigraphStopCommand + + +class StopCommand(QoxigraphStopCommand): + STATUS_COMMAND = StatusCommand() + DEFAULT_REGEX = "java\\s+-server.*=%%PORT%%.*blazegraph.jar" + + def description(self) -> str: + return "Stop Blazegraph server for a given dataset or port" + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + # Add port in the server section to the arguments + args = super().relevant_qleverfile_arguments() + args.setdefault("server", []).append("port") + return args + + def execute(self, args) -> bool: + args.cmdline_regex = args.cmdline_regex.replace( + "%%PORT%%", str(args.port) + ) + return super().execute(args) diff --git a/src/qblazegraph/qleverfile.py b/src/qblazegraph/qleverfile.py new file mode 100644 index 00000000..e4d45008 --- /dev/null +++ b/src/qblazegraph/qleverfile.py @@ -0,0 +1,62 @@ +from __future__ import annotations + + +def qleverfile_args(all_args: dict[str, dict[str, tuple]]) -> None: + """Define additional blazegraph specific Qleverfile parameters""" + + def arg(*args, **kwargs): + return (args, kwargs) + + index_args = all_args["index"] + server_args = all_args["server"] + + index_args["jvm_args"] = arg( + "--jvm_args", + type=str, + default="-Xmx4G", + help=( + "Arguments for the JVM. " + "Do not set to all available RAM. " + "Increasing is only necessary for large numbers of long literals." + ), + ) + index_args["extra_args"] = arg( + "--extra-args", + type=str, + default="", + help=( + "Additional arguments to pass directly to the Blazegraph BulkLoader. " + "This allows advanced users to specify options not exposed in " + "Qleverfile. The string is appended verbatim to the command." + ), + ) + + server_args["jvm_args"] = arg( + "--jvm_args", + type=str, + default="-Xmx4G", + help=("Arguments for the JVM."), + ) + server_args["timeout"] = arg( + "--timeout", + type=str, + default="60s", + help="The maximal time in seconds a query is allowed to run", + ) + server_args["read_only"] = arg( + "--read-only", + type=str, + choices=["yes", "no"], + default="yes", + help="The REST API will not permit mutation operations in read-only mode.", + ) + server_args["extra_args"] = arg( + "--extra-args", + type=str, + default="", + help=( + "Additional -D props to pass directly to the java -jar blazegraph.jar. " + "This allows advanced users to specify options not exposed in " + "Qleverfile. The string is appended verbatim to the command." + ), + ) diff --git a/src/qgraphdb/__init__.py b/src/qgraphdb/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qgraphdb/commands/__init__.py b/src/qgraphdb/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qgraphdb/commands/benchmark_queries.py b/src/qgraphdb/commands/benchmark_queries.py new file mode 100644 index 00000000..349dec2d --- /dev/null +++ b/src/qgraphdb/commands/benchmark_queries.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from qlever.commands.benchmark_queries import ( + BenchmarkQueriesCommand as QleverBenchmarkQueriesCommand, +) + + +class BenchmarkQueriesCommand(QleverBenchmarkQueriesCommand): + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + qleverfile_args = super().relevant_qleverfile_arguments() + if qleverfile_args.get("data"): + qleverfile_args["data"].append("name") + else: + qleverfile_args["data"] = ["name"] + return qleverfile_args + + def execute(self, args) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = ( + f"{args.host_name}:{args.port}/repositories/{args.name}" + ) + return super().execute(args) diff --git a/src/qgraphdb/commands/get_data.py b/src/qgraphdb/commands/get_data.py new file mode 120000 index 00000000..4900dbb8 --- /dev/null +++ b/src/qgraphdb/commands/get_data.py @@ -0,0 +1 @@ +../../qlever/commands/get_data.py \ No newline at end of file diff --git a/src/qgraphdb/commands/index.py b/src/qgraphdb/commands/index.py new file mode 100644 index 00000000..5bf18dd4 --- /dev/null +++ b/src/qgraphdb/commands/index.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from pathlib import Path + +import rdflib + +import qlever.util as util +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log + + +class IndexCommand(QleverCommand): + def __init__(self): + self.script_name = "qgraphdb" + + def description(self) -> str: + return "Build the index for a given RDF dataset" + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name", "format"], + "index": [ + "input_files", + "index_binary", + "threads", + "jvm_args", + "entity_index_size", + "ruleset", + "extra_args", + ], + "server": ["timeout", "read_only"], + "runtime": [ + "system", + "image", + "index_container", + ], + } + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "--overwrite-existing", + action="store_true", + default=False, + help="Overwrite an existing index, think twice before using this", + ) + + def update_config_ttl(self, config_dict: dict[str, str]) -> None: + graph = rdflib.Graph() + graph.parse(Path.cwd() / "config.ttl", format="ttl") + for sub, pred, obj in graph: + pred_str = str(pred).split("#")[1] + if pred_str in config_dict: + new_val = rdflib.Literal(config_dict[pred_str]) + graph.remove((sub, pred, obj)) + graph.add((sub, pred, new_val)) + graph.serialize(destination=Path.cwd() / "config.ttl", format="ttl") + log.info( + "config.ttl successfully overwritten with relevant Qleverfile entries!" + ) + + @staticmethod + def construct_config_ttl_dict(args) -> dict[str, str]: + try: + timeout = int(args.timeout[:-1]) + except ValueError: + timeout = 0 + + config_dict = { + "repositoryID": args.name, + "label": f"{args.name} repository TTL config file", + "throw-QueryEvaluationException-on-timeout": "true", + "read-only": "true" if args.read_only == "yes" else "false", + "query-timeout": str(timeout), + "ruleset": args.ruleset, + "entity-index-size": str(args.entity_index_size), + } + + return config_dict + + @staticmethod + def wrap_cmd_in_container(args, cmd: str) -> str: + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand="run --rm", + image_name=args.image, + container_name=args.index_container, + volumes=[("$(pwd)", "/opt/graphdb/home")], + working_directory="/opt/graphdb/home", + ) + + def execute(self, args, called_from_conformance_test: bool = False) -> bool: + index_cmd = ( + f"{args.index_binary} preload {args.jvm_args} -c config.ttl" + ) + if args.overwrite_existing: + index_cmd += " -f" + if args.threads: + index_cmd += f" -t {args.threads}" + index_cmd += f" -Dgraphdb.home={args.name}_index {args.extra_args} " + if called_from_conformance_test: + index_cmd += ( + f"{args.input_files} > {args.name}.index-log.txt 2>&1" + ) + else: + index_cmd += ( + f"{args.input_files} | tee {args.name}.index-log.txt" + ) + + if args.system != "native": + index_cmd = self.wrap_cmd_in_container(args, index_cmd) + + config_dict = self.construct_config_ttl_dict(args) + if not called_from_conformance_test: + log.info( + "Following options of GraphDB config.ttl will be updated " + "with the values from Qleverfile as shown below:\n" + ) + for option, value in config_dict.items(): + log.info(f"{option} = {value}") + log.info("") + + # Show the command line. + self.show(index_cmd, only_show=args.show) + if args.show: + if not Path("config.ttl").exists(): + log.warning( + "config.ttl file not found in current working directory! " + "The index command will fail in its absence!" + ) + return True + + # Check if all of the input files exist. + if not util.input_files_exist(args.input_files): + return False + + # When running natively, check if the binary exists and works. + if args.system == "native": + if not util.binary_exists(args.index_binary, "index-binary", args): + return False + else: + if Containerize().is_running(args.system, args.index_container): + log.info( + f"{args.system} container {args.index_container} is still up, " + "which means that data loading is in progress. Please wait..." + ) + return False + # A previously interrupted run can leave a container with this name + # in a non-running (Created/Exited) state, which makes the + # `docker/podman run --name` below fail with exit code 125. Remove + # any such stale container first. + Containerize.stop_and_remove_container( + args.system, args.index_container + ) + + if not Path("config.ttl").exists(): + log.error( + "config.ttl file not found in the current working directory! " + f"Did you call {self.script_name} setup-config {args.name}?" + ) + return False + + index_dir = Path(f"{args.name}_index/data/repositories/{args.name}") + if index_dir.exists() and not args.overwrite_existing: + log.error( + f'Index files for basename "{args.name}" found, if you ' + f"want to overwrite them, use --overwrite-existing" + ) + log.info("") + log.info( + f"Index directory found in current directory: {index_dir}" + ) + return False + + # Run the index command. + try: + self.update_config_ttl(config_dict) + util.run_command( + index_cmd, show_output=not called_from_conformance_test + ) + except Exception as e: + log.error(f"Building the index failed: {e}") + return False + + return True diff --git a/src/qgraphdb/commands/index_stats.py b/src/qgraphdb/commands/index_stats.py new file mode 100644 index 00000000..4a8c50ed --- /dev/null +++ b/src/qgraphdb/commands/index_stats.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import re + +from qlever.commands.index_stats import ( + IndexStatsCommand as QleverIndexStatsCommand, +) +from qlever.log import log +from qlever.util import get_total_file_size, run_command + + +class IndexStatsCommand(QleverIndexStatsCommand): + """ + Class for executing the `index-stats` command. + """ + + def execute_time( + self, args, log_file_name: str + ) -> dict[str, tuple[float | None, str]]: + # Read the last few lines of the log file (the total time is + # always near the end). + try: + log_text = run_command( + f"tail {log_file_name}", return_output=True + ) + except Exception as e: + log.error(f"Problem reading index log file {log_file_name}: {e}") + return {} + + stats = {} + # Pattern: "Finished in ms" (total time, always last) + total_pattern = re.compile(r"Finished in ([\d,]+)ms\s*$") + + for line in log_text.splitlines(): + match = total_pattern.search(line) + if not match: + continue + + try: + value_s = float(match.group(1).replace(",", "")) / 1000 + except (ValueError, TypeError): + continue + + time_unit = self.get_time_unit(args.time_unit, value_s) + unit_factor = self.get_time_unit_factor(time_unit) + + stats["TOTAL time"] = (value_s / unit_factor, time_unit) + break + + return stats + + def execute_space(self, args) -> dict[str, tuple[float, str]]: + """ + Part of `execute` that returns the space used by different types of + index along with the unit. + """ + storage = f"{args.name}_index/data/repositories/{args.name}/storage" + index_size = get_total_file_size( + [f"{storage}/*", f"{storage}/literals-index/*"], + exclude={"owlim.properties", "rule.list", "last_precommit_id"}, + ) + + size_unit = self.get_size_unit(args.size_unit, index_size) + unit_factor = self.get_size_unit_factor(size_unit) + + index_size /= unit_factor + + return {"TOTAL size": (index_size, size_unit)} diff --git a/src/qgraphdb/commands/log.py b/src/qgraphdb/commands/log.py new file mode 120000 index 00000000..90ffabb4 --- /dev/null +++ b/src/qgraphdb/commands/log.py @@ -0,0 +1 @@ +../../qlever/commands/log.py \ No newline at end of file diff --git a/src/qgraphdb/commands/query.py b/src/qgraphdb/commands/query.py new file mode 100644 index 00000000..c7180d9d --- /dev/null +++ b/src/qgraphdb/commands/query.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from qoxigraph.commands.query import QueryCommand as QoxigraphQueryCommand + + +class QueryCommand(QoxigraphQueryCommand): + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + qleverfile_args = super().relevant_qleverfile_arguments() + if qleverfile_args.get("data"): + qleverfile_args["data"].append("name") + else: + qleverfile_args["data"] = ["name"] + return qleverfile_args + + def execute(self, args, called_from_conformance_test: bool = False) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = ( + f"{args.host_name}:{args.port}/repositories/{args.name}" + ) + return super().execute( + args, called_from_conformance_test=called_from_conformance_test + ) diff --git a/src/qgraphdb/commands/setup_config.py b/src/qgraphdb/commands/setup_config.py new file mode 100644 index 00000000..6b32d8df --- /dev/null +++ b/src/qgraphdb/commands/setup_config.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from qlever.log import log +from qlever.util import add_memory_options, run_curl_command +from qoxigraph.commands.setup_config import ( + SetupConfigCommand as QoxigraphSetupConfigCommand, +) + + +class SetupConfigCommand(QoxigraphSetupConfigCommand): + """ + Should behave exactly the same as setup-config command in qoxigraph, + just with a different Docker image name + """ + + IMAGE = "docker.io/ontotext/graphdb:11.2.1" + + def additional_arguments(self, subparser) -> None: + super().additional_arguments(subparser) + add_memory_options(subparser) + + @staticmethod + def construct_engine_specific_params(args) -> dict[str, dict[str, str]]: + index_memory = int(args.total_index_memory[:-1]) + server_memory = int(args.total_server_memory[:-1]) + entity_index_size = min(2_140_000_000, (10_000_000 * index_memory * 3)) + return { + "index": { + "ENTITY_INDEX_SIZE": entity_index_size, + "RULESET": "empty", + "JVM_ARGS": f"-Xms{index_memory}G -Xmx{index_memory}G", + }, + "server": {"HEAP_SIZE_GB": f"{server_memory}G", "TIMEOUT": "60s"}, + } + + def execute(self, args) -> bool: + qleverfile_successfully_created = super().execute(args) + if not qleverfile_successfully_created: + return False + + repo_config_ttl = ( + "https://graphdb.ontotext.com/documentation/11.0/_downloads/" + "565be93599bf4c3324147fb94b562595/repo-config.ttl" + ) + try: + run_curl_command(url=repo_config_ttl, result_file="config.ttl") + log.info( + "Successfully downloaded config.ttl for specifying config " + "parameters. There is no need to change this file as all the " + "relevant entries will be overwritten by entries from the Qleverfile." + ) + return True + except Exception as e: + log.error(f"Failed to download config.ttl file: {e}") + log.info(f"Download it manually from {repo_config_ttl}") + log.info( + "There is no need to change this file after downloading it as " + "all the relevant entries will be overwritten by entries from " + "the Qleverfile." + ) + return False diff --git a/src/qgraphdb/commands/start.py b/src/qgraphdb/commands/start.py new file mode 100644 index 00000000..4cf5a95c --- /dev/null +++ b/src/qgraphdb/commands/start.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import subprocess +import time +from pathlib import Path + +import qlever.util as util +from qgraphdb.commands.stop import StopCommand +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log + + +class StartCommand(QleverCommand): + def __init__(self): + self.script_name = "qgraphdb" + + def description(self) -> str: + return ( + "Start the server for GraphDB (requires that you have built an " + "index before)" + ) + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "server": [ + "host_name", + "heap_size_gb", + "server_binary", + "port", + "extra_env_args", + "extra_args", + ], + "runtime": [ + "system", + "image", + "server_container", + "license_file_path", + ], + } + + def additional_arguments(self, subparser): + subparser.add_argument( + "--run-in-foreground", + action="store_true", + default=False, + help=( + "Run the start command in the foreground " + "(default: run in the background)" + ), + ) + + @staticmethod + def wrap_cmd_in_container(args, cmd: str) -> str: + run_subcommand = "run --restart=unless-stopped" + if not args.run_in_foreground: + run_subcommand += " -d" + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand=run_subcommand, + image_name=args.image, + container_name=args.server_container, + volumes=[ + ("$(pwd)", "/opt/graphdb/home"), + ( + str(args.license_file_path.resolve()), + "/opt/graphdb/graphdb.license", + ), + ], + working_directory="/opt/graphdb/home", + ports=[(args.port, args.port)], + ) + + def execute(self, args, called_from_conformance_test: bool = False) -> bool: + license_file_path = ( + str(args.license_file_path.resolve()) + if args.system == "native" + else "/opt/graphdb/graphdb.license" + ) + start_cmd = ( + f'env GDB_HEAP_SIZE="{args.heap_size_gb}" {args.extra_env_args} ' + f"{args.server_binary} -s -Dgraphdb.home={args.name}_index " + f"-Dgraphdb.connector.port={args.port} {args.extra_args} " + f"-Dgraphdb.license.file={license_file_path}" + f" > {args.name}.server-log.txt 2>&1" + ) + + if args.system == "native": + if not args.run_in_foreground: + start_cmd = f"nohup {start_cmd} &" + else: + start_cmd = self.wrap_cmd_in_container(args, start_cmd) + + # Show the command line. + self.show(start_cmd, only_show=args.show) + if args.show: + return True + + # When running natively, check if the binary exists and works. + if args.system == "native": + if not util.binary_exists(args.server_binary, "server-binary", args): + return False + + endpoint_url = f"http://{args.host_name}:{args.port}/repositories" + if util.is_server_alive(url=endpoint_url): + log.error(f"GraphDB server already running on {endpoint_url}\n") + log.info( + f"To kill the existing server, use `{self.script_name} stop`" + ) + return False + + try: + process = util.run_command( + start_cmd, + use_popen=args.run_in_foreground, + ) + except Exception as e: + log.error(f"Starting the GraphDB server failed ({e})") + return False + + log_proc = None + # Tail the server log until the server is ready (note that the `exec` + # is important to make sure that the tail process is killed and not + # just the bash process). + if not called_from_conformance_test: + if args.run_in_foreground: + log.info( + "Follow the server logs as long as the server is" + " running (Ctrl-C stops the server)" + ) + else: + log.info( + "Follow the server logs until the server is ready" + " (Ctrl-C stops following the log, but NOT the server)" + ) + log.info("") + log_file = Path(f"{args.name}.server-log.txt") + while not log_file.exists(): + time.sleep(0.1) + log_cmd = f"exec tail -f {log_file}" + log_proc = subprocess.Popen(log_cmd, shell=True) + while not util.is_server_alive(endpoint_url): + time.sleep(1) + + if not called_from_conformance_test: + log.info(f"GraphDB sparql endpoint for queries is {endpoint_url}") + + # Kill the log process + if not args.run_in_foreground and log_proc: + log_proc.terminate() + + # With `--run-in-foreground`, wait until the server is stopped. + if args.run_in_foreground: + try: + process.wait() + except KeyboardInterrupt: + process.terminate() + if args.system in Containerize.supported_systems(): + args.cmdline_regex = StopCommand.DEFAULT_REGEX + StopCommand().execute(args) + if log_proc: + log_proc.terminate() + + return True diff --git a/src/qgraphdb/commands/status.py b/src/qgraphdb/commands/status.py new file mode 100644 index 00000000..ee518edf --- /dev/null +++ b/src/qgraphdb/commands/status.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from qoxigraph.commands.status import StatusCommand as QoxigraphStatusCommand + + +class StatusCommand(QoxigraphStatusCommand): + DEFAULT_REGEX = "com.ontotext.graphdb.server.GraphDBServer" + + def description(self) -> str: + return "Show GraphDB process running on this machine" diff --git a/src/qgraphdb/commands/stop.py b/src/qgraphdb/commands/stop.py new file mode 100644 index 00000000..a319773f --- /dev/null +++ b/src/qgraphdb/commands/stop.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from qjena.commands.status import StatusCommand +from qoxigraph.commands.stop import StopCommand as QoxigraphStopCommand + + +class StopCommand(QoxigraphStopCommand): + STATUS_COMMAND = StatusCommand() + DEFAULT_REGEX = r".*-Dgraphdb\.home=%%NAME%%\S*.*GraphDBServer$" + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "runtime": ["system", "server_container"], + } + + def description(self) -> str: + return "Stop the GraphDB server" + + def execute(self, args) -> bool: + args.cmdline_regex = args.cmdline_regex.replace("%%NAME%%", args.name) + return super().execute(args) diff --git a/src/qgraphdb/qleverfile.py b/src/qgraphdb/qleverfile.py new file mode 100644 index 00000000..dd5e1efa --- /dev/null +++ b/src/qgraphdb/qleverfile.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from pathlib import Path + + +def qleverfile_args(all_args: dict[str, dict[str, tuple]]) -> None: + """Define additional jena specific Qleverfile parameters""" + + def arg(*args, **kwargs): + return (args, kwargs) + + index_args = all_args["index"] + server_args = all_args["server"] + runtime_args = all_args["runtime"] + + index_args["index_binary"] = arg( + "--index-binary", + type=str, + default="importrdf", + help=( + "The binary for building the index (default: importrdf) " + "(this requires that you have GraphDB installed " + "on your machine)" + ), + ) + index_args["threads"] = arg( + "--threads", + type=int, + default=None, + help=("Number of rdf parsers."), + ) + index_args["entity_index_size"] = arg( + "--entity-index-size", + type=int, + default=10000000, + help=( + "Defines the initial size of the entity hash table index entries. " + "The bigger the size, the fewer the collisions in the hash table, " + "and the faster the entity retrieval. The entity hash table will " + "adapt to the number of stored entities once the number of collisions " + "passes a critical threshold." + ), + ) + index_args["jvm_args"] = arg( + "--jvm_args", + type=str, + default="-Xms4g -Xmx4G", + help=("Arguments for the JVM. "), + ) + index_args["ruleset"] = arg( + "--ruleset", + type=str, + default="empty", + choices=[ + "empty", + "rdfs", + "owl-horst", + "owl-max", + "owl2-rl", + "rdfs-optimized", + "owl-horst-optimized", + "owl-max-optimized", + "owl2-rl-optimized", + ], + help=( + "Sets of axiomatic triples, consistency checks and entailment rules, " + "which determine the applied semantics." + ), + ) + index_args["extra_args"] = arg( + "--extra-args", + type=str, + default="", + help=( + "Additional arguments to pass directly to the importrdf binary. " + "This allows advanced users to specify options not exposed in " + "Qleverfile. The string is appended verbatim to the command." + ), + ) + + server_args["heap_size_gb"] = arg( + "--heap_size_gb", + type=str, + default="4G", + help=( + "Sets the Java minimum and maximum heap size (-Xms and -Xmx option)." + ), + ) + server_args["server_binary"] = arg( + "--server-binary", + type=str, + default="graphdb", + help=( + "The binary for starting the server (default: graphdb) " + "(this requires that you have GraphDB installed " + "on your machine)" + ), + ) + server_args["timeout"] = arg( + "--timeout", + type=str, + default="30s", + help="The maximal time in seconds a query is allowed to run", + ) + server_args["read_only"] = arg( + "--read-only", + type=str, + choices=["yes", "no"], + default="yes", + help=( + "The HTTP server will not permit mutation operations in " + "read-only mode" + ), + ) + server_args["extra_env_args"] = arg( + "--extra-env-args", + type=str, + default="", + help=( + "Additional environment variable arguments to pass directly to " + "the graphdb binary as a string of key=value pairs. This allows " + "advanced users to specify options not exposed in Qleverfile. " + "The string is appended verbatim to the bash env command." + ), + ) + server_args["extra_args"] = arg( + "--extra-args", + type=str, + default="", + help=( + "Additional arguments to pass directly to the graphdb binary. " + "This allows advanced users to specify options not exposed in " + "Qleverfile. The string is appended verbatim to the command." + ), + ) + + runtime_args["license_file_path"] = arg( + "--license-file-path", + type=Path, + required=True, + help=( + "Path to the GraphDB license file. Get the free license file from " + "https://www.ontotext.com/products/graphdb/#try-graphdb" + ) + ) diff --git a/src/qjena/Dockerfile b/src/qjena/Dockerfile new file mode 100644 index 00000000..aa7c99e2 --- /dev/null +++ b/src/qjena/Dockerfile @@ -0,0 +1,40 @@ +# Use an official OpenJDK runtime as a parent image +FROM docker.io/library/openjdk:26-ea-21-jdk-slim + +RUN apt-get update && apt-get install -y wget unzip jq + +WORKDIR /opt + +# Download and extract Apache Jena Fuseki and Apache Jena using the latest version dynamically +RUN LATEST_VERSION=$(wget -qO- https://dlcdn.apache.org/jena/binaries/ \ + | grep -oP 'apache-jena-\K[0-9]+\.[0-9]+\.[0-9]+' \ + | sort -V \ + | tail -n 1) \ + && wget https://dlcdn.apache.org/jena/binaries/apache-jena-fuseki-${LATEST_VERSION}.zip \ + && unzip apache-jena-fuseki-${LATEST_VERSION}.zip \ + && rm -f apache-jena-fuseki-${LATEST_VERSION}.zip \ + && wget https://dlcdn.apache.org/jena/binaries/apache-jena-${LATEST_VERSION}.zip \ + && unzip apache-jena-${LATEST_VERSION}.zip \ + && rm -f apache-jena-${LATEST_VERSION}.zip \ + && mv apache-jena-${LATEST_VERSION} /opt/apache-jena \ + && mv apache-jena-fuseki-${LATEST_VERSION} /opt/apache-jena-fuseki + +# Set ownership to the user passed by UID and GID +ARG UID +ARG GID +RUN if [ "${UID:-}" != "" ] && [ "${GID:-}" != "" ]; then \ + chown -R ${UID}:${GID} /opt; \ + fi + +# Ensure the bin and fuseki folders are in PATH +ENV JENA_HOME="/opt/apache-jena" +ENV FUSEKI_HOME="/opt/apache-jena-fuseki" +ENV PATH="${JENA_HOME}/bin:${FUSEKI_HOME}:${PATH}" + +# Make sure scripts are executable +RUN chmod +x /opt/apache-jena/bin/* + +RUN chmod +x /opt/apache-jena-fuseki/fuseki-server.jar + +# Set entrypoint +CMD ["bash"] diff --git a/src/qjena/__init__.py b/src/qjena/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qjena/commands/__init__.py b/src/qjena/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qjena/commands/benchmark_queries.py b/src/qjena/commands/benchmark_queries.py new file mode 100644 index 00000000..0fba08ab --- /dev/null +++ b/src/qjena/commands/benchmark_queries.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from qlever.commands.benchmark_queries import ( + BenchmarkQueriesCommand as QleverBenchmarkQueriesCommand, +) + + +class BenchmarkQueriesCommand(QleverBenchmarkQueriesCommand): + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "server": ["host_name", "port", "timeout"], + "ui": ["ui_config"], + } + + def execute(self, args) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = ( + f"{args.host_name}:{args.port}/{args.name}/query" + ) + return super().execute(args) diff --git a/src/qjena/commands/get_data.py b/src/qjena/commands/get_data.py new file mode 120000 index 00000000..4900dbb8 --- /dev/null +++ b/src/qjena/commands/get_data.py @@ -0,0 +1 @@ +../../qlever/commands/get_data.py \ No newline at end of file diff --git a/src/qjena/commands/index.py b/src/qjena/commands/index.py new file mode 100644 index 00000000..9202cb82 --- /dev/null +++ b/src/qjena/commands/index.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from pathlib import Path + +import qlever.util as util +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log + + +class IndexCommand(QleverCommand): + def __init__(self): + self.script_name = "qjena" + + def description(self) -> str: + return "Build the index for a given RDF dataset" + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name", "format"], + "index": [ + "input_files", + "index_binary", + "threads", + "jvm_args", + "extra_args", + "extra_env_args", + ], + "runtime": ["system", "image", "index_container"], + } + + def additional_arguments(self, subparser): + subparser.add_argument( + "--rebuild-image", + action="store_true", + default=False, + help="Rebuild the Docker image to get the latest updates", + ) + + @staticmethod + def wrap_cmd_in_container(args, cmd: str) -> str: + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand="run --rm", + image_name=args.image, + container_name=args.index_container, + volumes=[("$(pwd)", "/opt/data")], + working_directory="/opt/data", + ) + + def execute(self, args, called_from_conformance_test: bool = False) -> bool: + system = args.system + input_files = args.input_files + + index_cmd = ( + f'env JVM_ARGS="{args.jvm_args}" {args.extra_env_args} ' + f"{args.index_binary} --threads {args.threads} --loc index " + f"{args.extra_args} {input_files}" + ) + if called_from_conformance_test: + index_cmd += f" > {args.name}.index-log.txt 2>&1" + else: + index_cmd += f" | tee {args.name}.index-log.txt" + + image_id = build_cmd = "" + if args.system == "native": + cmd_to_show = index_cmd + else: + index_cmd = self.wrap_cmd_in_container(args, index_cmd) + dockerfile_dir = Path(__file__).parent.parent + dockerfile_path = dockerfile_dir / "Dockerfile" + build_cmd = ( + f"{system} build -f {dockerfile_path} -t {args.image} --build-arg " + f"UID=$(id -u) --build-arg GID=$(id -g) {dockerfile_dir}" + ) + image_id = util.get_container_image_id(system, args.image) + cmd_to_show = ( + f"{build_cmd}\n\n{index_cmd}" + if not image_id or args.rebuild_image + else index_cmd + ) + + # Show the command line. + self.show(cmd_to_show, only_show=args.show) + if args.show: + return True + + # Check if all of the input files exist. + if not called_from_conformance_test and not util.input_files_exist( + input_files + ): + return False + + # When running natively, check if the binary exists and works. + if args.system == "native": + if not util.binary_exists(args.index_binary, "index-binary", args): + return False + else: + if Containerize().is_running(args.system, args.index_container): + log.info( + f"{args.system} container {args.index_container} is still up, " + "which means that data loading is in progress. Please wait..." + ) + return False + # A previously interrupted run can leave a container with this name + # in a non-running (Created/Exited) state, which makes the + # `docker/podman run --name` below fail with exit code 125. Remove + # any such stale container first. + Containerize.stop_and_remove_container( + args.system, args.index_container + ) + + if not image_id or args.rebuild_image: + build_successful = util.build_image( + build_cmd, system, args.image + ) + if not build_successful: + return False + else: + log.info(f"{args.image} image present on the system\n") + + index_dir = Path("index/Data-0001") + if index_dir.exists() and any(index_dir.iterdir()): + log.error( + "Index files found in index/Data-0001 directory " + "which shows presence of a previous index\n" + ) + log.info("Aborting the index operation...") + return False + + # Run the index command. + try: + util.run_command( + index_cmd, show_output=not called_from_conformance_test + ) + except Exception as e: + log.error(f"Building the index failed: {e}") + return False + + return True diff --git a/src/qjena/commands/index_stats.py b/src/qjena/commands/index_stats.py new file mode 100644 index 00000000..86ff1d81 --- /dev/null +++ b/src/qjena/commands/index_stats.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import re + +from qlever.commands.index_stats import ( + IndexStatsCommand as QleverIndexStatsCommand, +) +from qlever.log import log +from qlever.util import get_total_file_size, run_command + + +class IndexStatsCommand(QleverIndexStatsCommand): + """ + Class for executing the `index-stats` command. + """ + + def execute_time( + self, args, log_file_name: str + ) -> dict[str, tuple[float | None, str]]: + """ + Part of `execute` that returns the time used for each part of indexing + along with the unit. + """ + + # Read the content of `log_file_name` into a list of lines. + try: + log_text = run_command( + f"tail -n 20 {log_file_name}", return_output=True + ) + except Exception as e: + log.error(f"Problem reading index log file {log_file_name}: {e}") + return {} + + stats = {} + # Pattern: "