diff --git a/pyproject.toml b/pyproject.toml index b053fe62..3b34ae74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,9 @@ Github = "https://github.com/ad-freiburg/qlever" [project.scripts] "qlever" = "qlever.qlever_main:main" +"qmdb" = "qlever.qlever_main:main" +"qoxigraph" = "qlever.qlever_main:main" +"qlever-old" = "qlever.qlever_old:main" [tool.setuptools] license-files = ["LICENSE"] diff --git a/src/qmdb/Dockerfile b/src/qmdb/Dockerfile new file mode 100644 index 00000000..eb515348 --- /dev/null +++ b/src/qmdb/Dockerfile @@ -0,0 +1,63 @@ +# Clone the MillenniumDB repository as the first step +FROM alpine:3.18 AS clone +WORKDIR /mdb-src + +RUN apk --no-cache add git +RUN git clone --depth 1 https://github.com/MillenniumDB/MillenniumDB.git . +RUN ls -l /mdb-src + +# Build stage +FROM alpine:3.18 AS build +WORKDIR /mdb + +# Install necessary build tools and dependencies +RUN apk --no-cache add cmake \ + make \ + g++ \ + openssl-dev \ + boost1.82-dev \ + ncurses-dev \ + icu-dev + +# Use files from the cloned repository +COPY --from=clone /mdb-src/src src +COPY --from=clone /mdb-src/CMakeLists.txt CMakeLists.txt +COPY --from=clone /mdb-src/third_party/antlr4-runtime-4.13.1 third_party/antlr4-runtime-4.13.1 + +# Build MillenniumDB +RUN cmake -B build -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=./ && \ + cmake --build build -j $(($(getconf _NPROCESSORS_ONLN)-1)) --target install + +COPY --from=clone /mdb-src/browser browser + +# Final minimal stage (to minimize image size) +FROM alpine:3.18 AS final +WORKDIR /data + +# Install runtime dependencies +RUN apk --no-cache add libstdc++ \ + libgcc \ + openssl \ + musl-locales \ + libncursesw \ + less \ + bash \ + icu-libs + +# Copy the binaries and browser from the build stage +COPY --from=build /mdb/build/bin /usr/bin +COPY --from=build /mdb/browser /browser + +# Set ownership to the user passed by UID and GID +ARG UID +ARG GID +RUN if [ "${UID:-}" != "" ] && [ "${GID:-}" != "" ]; then \ + chown -R ${UID}:${GID} /data; \ + fi + +# Expose necessary volumes and environment variables +VOLUME /data +ENV MDB_BROWSER=/browser + +# Default command to run in the container +CMD ["bash"] diff --git a/src/qmdb/__init__.py b/src/qmdb/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qmdb/commands/__init__.py b/src/qmdb/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qmdb/commands/benchmark_queries.py b/src/qmdb/commands/benchmark_queries.py new file mode 100644 index 00000000..c3961f37 --- /dev/null +++ b/src/qmdb/commands/benchmark_queries.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from qlever.commands.benchmark_queries import ( + BenchmarkQueriesCommand as QleverBenchmarkQueriesCommand, +) + + +class BenchmarkQueriesCommand(QleverBenchmarkQueriesCommand): + """ + Run benchmark queries against the MillenniumDB SPARQL endpoint. + Overrides the default endpoint to use MillenniumDB's /sparql path. + """ + + def execute(self, args) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = f"{args.host_name}:{args.port}/sparql" + return super().execute(args) diff --git a/src/qmdb/commands/example_queries.py b/src/qmdb/commands/example_queries.py new file mode 100644 index 00000000..d612b709 --- /dev/null +++ b/src/qmdb/commands/example_queries.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from qlever.commands.example_queries import ( + ExampleQueriesCommand as QleverExampleQueriesCommand, +) + + +class ExampleQueriesCommand(QleverExampleQueriesCommand): + def execute(self, args) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = f"{args.host_name}:{args.port}/sparql" + return super().execute(args) diff --git a/src/qmdb/commands/extract_queries.py b/src/qmdb/commands/extract_queries.py new file mode 120000 index 00000000..5667cc52 --- /dev/null +++ b/src/qmdb/commands/extract_queries.py @@ -0,0 +1 @@ +../../qlever/commands/extract_queries.py \ No newline at end of file diff --git a/src/qmdb/commands/get_data.py b/src/qmdb/commands/get_data.py new file mode 120000 index 00000000..4900dbb8 --- /dev/null +++ b/src/qmdb/commands/get_data.py @@ -0,0 +1 @@ +../../qlever/commands/get_data.py \ No newline at end of file diff --git a/src/qmdb/commands/index.py b/src/qmdb/commands/index.py new file mode 100644 index 00000000..7db60a03 --- /dev/null +++ b/src/qmdb/commands/index.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from pathlib import Path + +import qlever.util as util +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log +from qlever.memory_monitor import MemoryMonitor + + +def wrap_cmd_in_container(args, cmd: str) -> str: + """ + Wrap an indexing command in a container that is automatically + removed after the process exits (--rm). + """ + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand="run --rm", + image_name=args.image, + container_name=args.index_container, + volumes=[("$(pwd)", "/data")], + working_directory="/data", + ) + + +class IndexCommand(QleverCommand): + """ + Build a MillenniumDB index for an RDF dataset. The indexing workflow is: + 1. Run `mdb import` to import input files into the index directory. + 2. For compressed data, pipe input through stdin with --format. + + Supports native and containerized execution. When using containers, + the Docker image is built from the MillenniumDB GitHub repository + if not already present. + """ + + def __init__(self): + pass + + def description(self) -> str: + return "Build the index for a given RDF dataset" + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name", "format"], + "index": [ + "input_files", + "cat_input_files", + "buffer_strings", + "buffer_tensors", + "btree_permutations", + "prefixes", + "extra_args", + ], + "runtime": ["system", "image", "index_container"], + } + + def additional_arguments(self, subparser): + subparser.add_argument( + "--index-binary", + type=str, + default="mdb", + help=( + "The binary for building the index (default: mdb) " + "(this requires that you have Millennium DB built from source " + "on your machine)" + ), + ) + subparser.add_argument( + "--rebuild-image", + action="store_true", + default=False, + help="Rebuild the Docker image to get the latest updates", + ) + + def execute(self, args) -> bool: + system = args.system + input_files = args.input_files + + # For compressed data, pipe the data from stdin with mandatory + # --format so MillenniumDB knows the RDF serialization. + if args.cat_input_files: + index_cmd = ( + f"{args.cat_input_files} | {args.index_binary} import " + f"{args.name}_index --format {args.format}" + ) + else: + index_cmd = ( + f"{args.index_binary} import {input_files} {args.name}_index" + ) + + # Append MillenniumDB-specific index options (btree permutations, + # buffer sizes, prefix compression). + index_cmd += f" --btree-permutations {args.btree_permutations}" + + if args.buffer_strings: + index_cmd += f" --buffer-strings {args.buffer_strings}B" + if args.buffer_tensors: + index_cmd += f" --buffer-tensors {args.buffer_tensors}B" + + if args.prefixes: + index_cmd += f" --prefixes {args.prefixes}" + if args.extra_args: + index_cmd += f" {args.extra_args}" + index_cmd += f" | tee {args.name}.index-log.txt" + + # For container execution, build the Docker image from the + # MillenniumDB repository if it is not already present. + image_id = build_cmd = "" + if args.system in Containerize.supported_systems(): + index_cmd = wrap_cmd_in_container(args, index_cmd) + dockerfile_url = "https://github.com/MillenniumDB/MillenniumDB.git" + build_cmd = f"{system} build {dockerfile_url} -t {args.image}" + + image_id = util.get_container_image_id(system, args.image) + + cmd_to_show = ( + f"{build_cmd}\n\n{index_cmd}" + if not image_id or args.rebuild_image + else index_cmd + ) + else: + cmd_to_show = index_cmd + + # Show the command line. + self.show(cmd_to_show, only_show=args.show) + if args.show: + return True + + # Check if all of the input files exist. + if not util.input_files_exist(input_files): + return False + + # Abort if a previous index already exists. Any files in the + # index directory indicate an existing store. + index_dir = Path(f"{args.name}_index") + if index_dir.exists() and any(index_dir.iterdir()): + log.error( + f"Index files found in {args.name}_index directory " + "which shows presence of a previous index\n" + ) + log.info("Aborting the index operation...") + return False + + if args.system in Containerize.supported_systems(): + if Containerize().is_running(args.system, args.index_container): + log.info( + f"{args.system} container {args.index_container} is still up, " + "which means that data loading is in progress. Please wait..." + ) + return False + + # Build the docker image if not found on the system + if not image_id or args.rebuild_image: + build_successful = util.build_image( + build_cmd, system, args.image + ) + if not build_successful: + return False + else: + log.info(f"{args.image} image present on the system\n") + else: + # When running natively, check if the binary exists and works. + if not util.binary_exists(args.index_binary, "index-binary", args): + return False + + # Run the index command. + try: + with MemoryMonitor( + dataset=args.name, + cmdline_regex=args.index_binary, + container=args.index_container, + system=args.system, + ): + util.run_command(index_cmd, show_output=True) + except Exception as e: + log.error(f"Building the index failed: {e}") + return False + + return True diff --git a/src/qmdb/commands/index_stats.py b/src/qmdb/commands/index_stats.py new file mode 100644 index 00000000..ee5b18e6 --- /dev/null +++ b/src/qmdb/commands/index_stats.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import re + +from qlever.commands.index_stats import ( + IndexStatsCommand as QleverIndexStatsCommand, +) +from qlever.commands.index_stats import ( + get_size_unit, + get_size_unit_factor, + get_time_unit, + get_time_unit_factor, +) +from qlever.log import log +from qlever.util import get_total_file_size + + +class IndexStatsCommand(QleverIndexStatsCommand): + """ + Show index build time and disk space usage for a MillenniumDB dataset. + Time is parsed from the "duration:" lines in the index log; space is + the total size of all files in the index directory. + """ + + def execute_time( + self, args, log_file_name: str + ) -> dict[str, tuple[float | None, str]]: + """ + Parse the MillenniumDB index log to extract build times. Each log + line matching "