Add qmdb (MillenniumDB) commands

pyproject.toml

@@ -27,6 +27,9 @@ Github = "https://github.com/ad-freiburg/qlever"
 
 [project.scripts]
 "qlever" = "qlever.qlever_main:main"
+"qmdb" = "qlever.qlever_main:main"
+"qoxigraph" = "qlever.qlever_main:main"
+"qlever-old" = "qlever.qlever_old:main"
 
 [tool.setuptools]
 license-files = ["LICENSE"]

src/qmdb/Dockerfile

@@ -0,0 +1,63 @@
+# Clone the MillenniumDB repository as the first step
+FROM alpine:3.18 AS clone
+WORKDIR /mdb-src
+
+RUN apk --no-cache add git
+RUN git clone --depth 1 https://github.com/MillenniumDB/MillenniumDB.git .
+RUN ls -l /mdb-src
+
+# Build stage
+FROM alpine:3.18 AS build
+WORKDIR /mdb
+
+# Install necessary build tools and dependencies
+RUN apk --no-cache add cmake \
+                       make \
+                       g++ \
+                       openssl-dev \
+                       boost1.82-dev \
+                       ncurses-dev \
+                       icu-dev
+
+# Use files from the cloned repository
+COPY --from=clone /mdb-src/src src
+COPY --from=clone /mdb-src/CMakeLists.txt CMakeLists.txt
+COPY --from=clone /mdb-src/third_party/antlr4-runtime-4.13.1 third_party/antlr4-runtime-4.13.1
+
+# Build MillenniumDB
+RUN cmake -B build -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=./ && \
+    cmake --build build -j $(($(getconf _NPROCESSORS_ONLN)-1)) --target install
+
+COPY --from=clone /mdb-src/browser browser
+
+# Final minimal stage (to minimize image size)
+FROM alpine:3.18 AS final
+WORKDIR /data
+
+# Install runtime dependencies
+RUN apk --no-cache add libstdc++ \
+                       libgcc \
+                       openssl \
+                       musl-locales \
+                       libncursesw \
+                       less \
+                       bash \
+                       icu-libs
+
+# Copy the binaries and browser from the build stage
+COPY --from=build /mdb/build/bin /usr/bin
+COPY --from=build /mdb/browser /browser
+
+# Set ownership to the user passed by UID and GID
+ARG UID
+ARG GID
+RUN if [ "${UID:-}" != "" ] && [ "${GID:-}" != "" ]; then \
+        chown -R ${UID}:${GID} /data; \
+    fi
+
+# Expose necessary volumes and environment variables
+VOLUME /data
+ENV MDB_BROWSER=/browser
+
+# Default command to run in the container
+CMD ["bash"]

src/qmdb/commands/benchmark_queries.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from qlever.commands.benchmark_queries import (
+    BenchmarkQueriesCommand as QleverBenchmarkQueriesCommand,
+)
+
+
+class BenchmarkQueriesCommand(QleverBenchmarkQueriesCommand):
+    """
+    Run benchmark queries against the MillenniumDB SPARQL endpoint.
+    Overrides the default endpoint to use MillenniumDB's /sparql path.
+    """
+
+    def execute(self, args) -> bool:
+        if not args.sparql_endpoint:
+            args.sparql_endpoint = f"{args.host_name}:{args.port}/sparql"
+        return super().execute(args)

src/qmdb/commands/example_queries.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from qlever.commands.example_queries import (
+    ExampleQueriesCommand as QleverExampleQueriesCommand,
+)
+
+
+class ExampleQueriesCommand(QleverExampleQueriesCommand):
+    def execute(self, args) -> bool:
+        if not args.sparql_endpoint:
+            args.sparql_endpoint = f"{args.host_name}:{args.port}/sparql"
+        return super().execute(args)

src/qmdb/commands/extract_queries.py

@@ -0,0 +1 @@
+../../qlever/commands/extract_queries.py

src/qmdb/commands/get_data.py

@@ -0,0 +1 @@
+../../qlever/commands/get_data.py

src/qmdb/commands/index.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import qlever.util as util
+from qlever.command import QleverCommand
+from qlever.containerize import Containerize
+from qlever.log import log
+from qlever.memory_monitor import MemoryMonitor
+
+
+def wrap_cmd_in_container(args, cmd: str) -> str:
+    """
+    Wrap an indexing command in a container that is automatically
+    removed after the process exits (--rm).
+    """
+    return Containerize().containerize_command(
+        cmd=cmd,
+        container_system=args.system,
+        run_subcommand="run --rm",
+        image_name=args.image,
+        container_name=args.index_container,
+        volumes=[("$(pwd)", "/data")],
+        working_directory="/data",
+    )
+
+
+class IndexCommand(QleverCommand):
+    """
+    Build a MillenniumDB index for an RDF dataset. The indexing workflow is:
+    1. Run `mdb import` to import input files into the index directory.
+    2. For compressed data, pipe input through stdin with --format.
+
+    Supports native and containerized execution. When using containers,
+    the Docker image is built from the MillenniumDB GitHub repository
+    if not already present.
+    """
+
+    def __init__(self):
+        pass
+
+    def description(self) -> str:
+        return "Build the index for a given RDF dataset"
+
+    def should_have_qleverfile(self) -> bool:
+        return True
+
+    def relevant_qleverfile_arguments(self) -> dict[str, list[str]]:
+        return {
+            "data": ["name", "format"],
+            "index": [
+                "input_files",
+                "cat_input_files",
+                "buffer_strings",
+                "buffer_tensors",
+                "btree_permutations",
+                "prefixes",
+                "extra_args",
+            ],
+            "runtime": ["system", "image", "index_container"],
+        }
+
+    def additional_arguments(self, subparser):
+        subparser.add_argument(
+            "--index-binary",
+            type=str,
+            default="mdb",
+            help=(
+                "The binary for building the index (default: mdb) "
+                "(this requires that you have Millennium DB built from source "
+                "on your machine)"
+            ),
+        )
+        subparser.add_argument(
+            "--rebuild-image",
+            action="store_true",
+            default=False,
+            help="Rebuild the Docker image to get the latest updates",
+        )
+
+    def execute(self, args) -> bool:
+        system = args.system
+        input_files = args.input_files
+
+        # For compressed data, pipe the data from stdin with mandatory
+        # --format so MillenniumDB knows the RDF serialization.
+        if args.cat_input_files:
+            index_cmd = (
+                f"{args.cat_input_files} | {args.index_binary} import "
+                f"{args.name}_index --format {args.format}"
+            )
+        else:
+            index_cmd = (
+                f"{args.index_binary} import {input_files} {args.name}_index"
+            )
+
+        # Append MillenniumDB-specific index options (btree permutations,
+        # buffer sizes, prefix compression).
+        index_cmd += f" --btree-permutations {args.btree_permutations}"
+
+        if args.buffer_strings:
+            index_cmd += f" --buffer-strings {args.buffer_strings}B"
+        if args.buffer_tensors:
+            index_cmd += f" --buffer-tensors {args.buffer_tensors}B"
+
+        if args.prefixes:
+            index_cmd += f" --prefixes {args.prefixes}"
+        if args.extra_args:
+            index_cmd += f" {args.extra_args}"
+        index_cmd += f" | tee {args.name}.index-log.txt"
+
+        # For container execution, build the Docker image from the
+        # MillenniumDB repository if it is not already present.
+        image_id = build_cmd = ""
+        if args.system in Containerize.supported_systems():
+            index_cmd = wrap_cmd_in_container(args, index_cmd)
+            dockerfile_url = "https://github.com/MillenniumDB/MillenniumDB.git"
+            build_cmd = f"{system} build {dockerfile_url} -t {args.image}"
+
+            image_id = util.get_container_image_id(system, args.image)
+
+            cmd_to_show = (
+                f"{build_cmd}\n\n{index_cmd}"
+                if not image_id or args.rebuild_image
+                else index_cmd
+            )
+        else:
+            cmd_to_show = index_cmd
+
+        # Show the command line.
+        self.show(cmd_to_show, only_show=args.show)
+        if args.show:
+            return True
+
+        # Check if all of the input files exist.
+        if not util.input_files_exist(input_files):
+            return False
+
+        # Abort if a previous index already exists. Any files in the
+        # index directory indicate an existing store.
+        index_dir = Path(f"{args.name}_index")
+        if index_dir.exists() and any(index_dir.iterdir()):
+            log.error(
+                f"Index files found in {args.name}_index directory "
+                "which shows presence of a previous index\n"
+            )
+            log.info("Aborting the index operation...")
+            return False
+
+        if args.system in Containerize.supported_systems():
+            if Containerize().is_running(args.system, args.index_container):
+                log.info(
+                    f"{args.system} container {args.index_container} is still up, "
+                    "which means that data loading is in progress. Please wait..."
+                )
+                return False
+
+            # Build the docker image if not found on the system
+            if not image_id or args.rebuild_image:
+                build_successful = util.build_image(
+                    build_cmd, system, args.image
+                )
+                if not build_successful:
+                    return False
+            else:
+                log.info(f"{args.image} image present on the system\n")
+        else:
+            # When running natively, check if the binary exists and works.
+            if not util.binary_exists(args.index_binary, "index-binary", args):
+                return False
+
+        # Run the index command.
+        try:
+            with MemoryMonitor(
+                dataset=args.name,
+                cmdline_regex=args.index_binary,
+                container=args.index_container,
+                system=args.system,
+            ):
+                util.run_command(index_cmd, show_output=True)
+        except Exception as e:
+            log.error(f"Building the index failed: {e}")
+            return False
+
+        return True

src/qmdb/commands/index_stats.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import re
+
+from qlever.commands.index_stats import (
+    IndexStatsCommand as QleverIndexStatsCommand,
+)
+from qlever.commands.index_stats import (
+    get_size_unit,
+    get_size_unit_factor,
+    get_time_unit,
+    get_time_unit_factor,
+)
+from qlever.log import log
+from qlever.util import get_total_file_size
+
+
+class IndexStatsCommand(QleverIndexStatsCommand):
+    """
+    Show index build time and disk space usage for a MillenniumDB dataset.
+    Time is parsed from the "duration:" lines in the index log; space is
+    the total size of all files in the index directory.
+    """
+
+    def execute_time(
+        self, args, log_file_name: str
+    ) -> dict[str, tuple[float | None, str]]:
+        """
+        Parse the MillenniumDB index log to extract build times. Each log
+        line matching "<label> duration: <value> <unit>" produces one entry.
+        The "total import" label is normalized to "TOTAL time".
+        """
+
+        # Read the content of `log_file_name` into a list of lines.
+        try:
+            with open(log_file_name, "r") as log_file:
+                lines = log_file.readlines()
+        except Exception as e:
+            log.error(f"Problem reading index log file {log_file_name}: {e}")
+            return {}
+
+        stats = {}
+        # Pattern: "<label> = <number> seconds"
+        pattern = re.compile(
+            r"^(.*?)\s*duration:\s*([\d.]+)\s*(milliseconds|seconds|minutes|hours)",
+            re.IGNORECASE,
+        )
+
+        unit_to_seconds = {
+            "milliseconds": 1 / 1000,
+            "seconds": 1,
+            "minutes": 60,
+            "hours": 3600,
+        }
+
+        for line in lines:
+            label = raw_value = mdb_time_unit = None
+            match = pattern.search(line)
+            if match:
+                label = match.group(1).strip()
+                if label.lower() == "total import":
+                    label = "TOTAL time"
+                raw_value = match.group(2)
+                mdb_time_unit = match.group(3).lower()
+
+            if raw_value is None or mdb_time_unit is None:
+                continue
+
+            try:
+                value = float(raw_value)
+            except (ValueError, TypeError):
+                continue
+
+            factor = unit_to_seconds.get(mdb_time_unit)
+            if factor is None:
+                continue
+
+            value_s = value * factor
+
+            time_unit = get_time_unit(args.time_unit, value_s)
+            unit_factor = get_time_unit_factor(time_unit)
+
+            normalized_value = value_s / unit_factor
+            stats[label] = (normalized_value, time_unit)
+
+        return stats
+
+    def execute_space(self, args) -> dict[str, tuple[float, str]]:
+        """
+        Return the space used by the index files (all files in the index
+        directory) along with the unit.
+        """
+        index_size = get_total_file_size([f"{args.name}_index/*"])
+
+        size_unit = get_size_unit(args.size_unit, index_size)
+        unit_factor = get_size_unit_factor(size_unit)
+
+        index_size /= unit_factor
+
+        return {"TOTAL size": (index_size, size_unit)}