diff --git a/.github/workflows/end-to-end-test-qoxigraph.yml b/.github/workflows/end-to-end-test-qoxigraph.yml new file mode 100644 index 00000000..5754469e --- /dev/null +++ b/.github/workflows/end-to-end-test-qoxigraph.yml @@ -0,0 +1,71 @@ +name: End-to-end test for qoxigraph + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + merge_group: + +permissions: + contents: read + +jobs: + qoxigraph-end-to-end-test: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + system: [native, docker, podman] + + steps: + - name: Checkout the repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install qoxigraph (qlever-control) with the plot extra + run: | + python -m pip install --upgrade pip + pip install -e ".[plot]" + + - name: Install the latest oxigraph release binary on PATH + if: matrix.system == 'native' + run: | + url=$(curl -sL https://api.github.com/repos/oxigraph/oxigraph/releases/latest \ + | grep -oE 'https://[^"]+x86_64_linux_gnu' | head -1) + echo "Downloading oxigraph binary from $url" + curl -sL "$url" -o oxigraph + chmod +x oxigraph + sudo mv oxigraph /usr/local/bin/oxigraph + oxigraph --version + + - name: Make sure the container runtime is available and pull the image + if: matrix.system != 'native' + run: | + command -v ${{matrix.system}} || (sudo apt-get update && sudo apt-get install -y ${{matrix.system}}) + ${{matrix.system}} --version + ${{matrix.system}} pull ghcr.io/oxigraph/oxigraph + + - name: End-to-end test on the imdb dataset (${{matrix.system}}) + run: | + export QLEVER_ARGCOMPLETE_ENABLED=1 + mkdir -p qoxigraph-e2e && cd qoxigraph-e2e + qoxigraph setup-config imdb --system ${{matrix.system}} + qoxigraph get-data + qoxigraph index + + echo "Checking that the resource-usage log and plot were produced" + test -s imdb.index.resource-usage-log.tsv + test -f imdb.resource-usage-plot.png + + qoxigraph index-stats + qoxigraph start + qoxigraph status + qoxigraph query "SELECT * WHERE { ?s ?p ?o } LIMIT 1" + qoxigraph stop + ls -lh diff --git a/pyproject.toml b/pyproject.toml index bcfea5e4..30d5fbcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ bugtracker = "https://github.com/ad-freiburg/qlever/issues" [project.scripts] "qlever" = "qlever.qlever_main:main" +"qoxigraph" = "qlever.qlever_main:main" [tool.setuptools] package-data = { "qlever" = ["Qleverfiles/*", "evaluation/www/*", "monitor_queries/*.tcss"] } diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index 0aef33ee..590b734f 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -18,13 +18,7 @@ ) -def render_usage_plot( - dataset: str, - stxxl_memory: str, - settings_json: str, - plot_max_points: int, - plot_only: bool, -) -> Path | None: +def render_usage_plot(args, plot_only: bool) -> Path | None: """Render the resource-usage plot. When the plotting libraries are missing, this is an error if the @@ -32,7 +26,7 @@ def render_usage_plot( how to get the plot at info level since the index build succeeded. """ try: - from qlever.resource_usage import usage_plot + from qlever.resource_usage.usage_plot import UsagePlot except ImportError: if plot_only: log.error( @@ -46,12 +40,9 @@ def render_usage_plot( "`qlever index --resource-usage-plot-only`." ) return None - return usage_plot.render_usage_plot( - dataset, - stxxl_memory=stxxl_memory, - settings_json=settings_json, - plot_max_points=plot_max_points, - ) + return UsagePlot( + args.name, args, plot_max_points=args.resource_usage_plot_max_points + ).render() class IndexCommand(QleverCommand): @@ -233,13 +224,7 @@ def execute(self, args) -> bool: # Render the resource-usage plot from the existing log without # rebuilding the index. if args.resource_usage_plot_only: - plot_path = render_usage_plot( - args.name, - stxxl_memory=args.stxxl_memory or "", - settings_json=args.settings_json, - plot_max_points=args.resource_usage_plot_max_points, - plot_only=True, - ) + plot_path = render_usage_plot(args, plot_only=True) if plot_path is None: return False log.info(f"Resource-usage plot saved to `{plot_path.name}`") @@ -407,13 +392,7 @@ def execute(self, args) -> bool: Path(f"{args.name}.index.resource-usage-log.tsv").exists() or Path(f"{args.name}.resource-usage-log.tsv").exists() ): - plot_path = render_usage_plot( - args.name, - stxxl_memory=args.stxxl_memory or "", - settings_json=args.settings_json, - plot_max_points=args.resource_usage_plot_max_points, - plot_only=False, - ) + plot_path = render_usage_plot(args, plot_only=False) if plot_path is not None: log.info(f"Resource-usage plot saved to `{plot_path.name}`") diff --git a/src/qlever/commands/setup_config.py b/src/qlever/commands/setup_config.py index a558a47f..1276aacd 100644 --- a/src/qlever/commands/setup_config.py +++ b/src/qlever/commands/setup_config.py @@ -1,6 +1,5 @@ from __future__ import annotations -import subprocess from os import environ from pathlib import Path @@ -49,6 +48,10 @@ def additional_arguments(self, subparser) -> None: choices=self.qleverfile_names, help="The name of the pre-configured Qleverfile to create", ) + # Override defaults to None so that arguments explicitly passed + # by the user can be told apart from plain defaults. + for section, arg_name in self.override_args: + subparser.set_defaults(**{arg_name: None}) def check_qleverfile_exists(self) -> bool: """Return True if a Qleverfile already exists (and log an error).""" @@ -75,37 +78,49 @@ def execute(self, args) -> bool: "(since inside the container, QLever should run natively)" ) log.info("") - # Construct the command line and show it. + # Build the updates for the copied Qleverfile. qleverfile_path = ( self.qleverfiles_path / f"Qleverfile.{args.config_name}" ) - setup_config_cmd = f"cat {qleverfile_path} | {util.get_ini_sed_cmd('server', 'ACCESS_TOKEN', util.get_random_string(12), True)}" + updates = { + "server": { + "ACCESS_TOKEN": (util.get_random_string(12), True), + }, + } if qlever_is_running_in_container: - setup_config_cmd += ( - f" | {util.get_ini_sed_cmd('runtime', 'SYSTEM', 'native')}" - ) + updates.setdefault("runtime", {})["SYSTEM"] = ("native", False) else: for section, arg_name in self.override_args: if arg_value := getattr(args, arg_name, None): - setup_config_cmd += f" | {util.get_ini_sed_cmd(section, arg_name.upper(), arg_value)}" + updates.setdefault(section, {})[arg_name.upper()] = ( + str(arg_value), + False, + ) - setup_config_cmd += "> Qleverfile" - self.show(setup_config_cmd, only_show=args.show) + # Show the changes that will be applied to the copied template. + show_lines = [ + f"Copy {qleverfile_path} to Qleverfile with the following changes:" + ] + for section, option_dict in updates.items(): + show_lines.append(f"\n[{section}]") + for option, (value, is_suffix) in option_dict.items(): + shown_value = ( + "*" * len(value) if option == "ACCESS_TOKEN" else value + ) + show_lines.append(f"{option} = {shown_value}") + self.show("\n".join(show_lines), only_show=args.show) if args.show: return True if self.check_qleverfile_exists(): return False - # Copy the Qleverfile to the current directory. + # Copy the Qleverfile to the current directory, with the updates + # applied. try: - subprocess.run( - setup_config_cmd, - shell=True, - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - ) + lines = qleverfile_path.read_text().splitlines() + result = util.update_ini_values(lines, updates) + Path("Qleverfile").write_text("\n".join(result) + "\n") except Exception as e: log.error( f'Could not copy "{qleverfile_path}" to current directory: {e}' diff --git a/src/qlever/resource_usage/resource_monitor.py b/src/qlever/resource_usage/resource_monitor.py new file mode 100644 index 00000000..f374fcb9 --- /dev/null +++ b/src/qlever/resource_usage/resource_monitor.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, fields +from pathlib import Path + +import psutil + +from qlever.containerize import Containerize +from qlever.log import log +from qlever.util import ( + container_memory_to_bytes, + find_process_by_binary, + run_command, +) + + +@dataclass +class Sample: + """One sample of elapsed time, memory (RSS), and CPU usage; None + fields are written as empty TSV columns.""" + + elapsed_s: float | None = None + rss: int | None = None + cpu_percent: float | None = None + + +def sample_to_tsv_row(sample: Sample) -> str: + """Format a Sample as a TSV row; None fields become empty columns.""" + values = [getattr(sample, field.name) for field in fields(sample)] + return "\t".join("" if v is None else str(v) for v in values) + "\n" + + +def sample_process(proc: psutil.Process) -> Sample: + """One RSS+CPU read from a psutil.Process; empty Sample on access errors.""" + try: + mem = proc.memory_info() + cpu_pct = proc.cpu_percent(interval=None) + except (psutil.NoSuchProcess, psutil.AccessDenied): + return Sample() + return Sample(rss=mem.rss, cpu_percent=cpu_pct) + + +def sample_container(system: str, container: str) -> Sample: + """One RSS+CPU read via ` stats --no-stream` on a named container.""" + try: + output = run_command( + f"{system} stats --no-stream" + f" --format '{{{{.MemUsage}}}}\t{{{{.CPUPerc}}}}'" + f" {container}", + return_output=True, + ) + memory_field, cpu_field = output.strip().split("\t") + used_memory = memory_field.split("/")[0].strip() + cpu_percent = float(cpu_field.strip().rstrip("%")) + return Sample( + rss=container_memory_to_bytes(used_memory), + cpu_percent=cpu_percent, + ) + except Exception: + return Sample() + + +class ResourceMonitor: + """ + Monitor resource usage (memory, CPU) of an index-building + process. Works in both native mode (via psutil) and container mode + (via docker/podman stats). + + Usage as a context manager: + + with ResourceMonitor(dataset="wikidata", binary="qlever-index"): + run_command(cmd, show_output=True) + + # For container mode: + with ResourceMonitor(dataset="wikidata", + binary="qlever-index", + container="qlever.index.wikidata", + system="docker"): + run_command(cmd, show_output=True) + """ + + def __init__( + self, + dataset: str, + binary: str, + container: str | None = None, + system: str | None = None, + interval: float = 1.0, + output_dir: Path | None = None, + parent_pid: int | None = None, + ): + """ + Args: + dataset: Name of the dataset being indexed. + binary: Name of the index executable, matched against the + descendant processes (native mode only). + container: Container name to sample; when set with `system`, + sampling uses `docker/podman stats` not psutil. + system: Container runtime ("docker" or "podman"). + interval: Seconds between samples. + output_dir: Directory for the TSV usage log file. + parent_pid: PID whose descendants are searched for the index + process. Defaults to the current process; pass a + different PID when the target re-parents away from + us. + """ + self.dataset = dataset + self.binary = binary + self.container = container + self.system = system + self.interval = interval + self.output_dir = output_dir or Path.cwd() + self.parent_pid = parent_pid + self.peak_rss = 0 + self.worker_proc = None + self.log_file = None + self.stop_event = threading.Event() + self.start_time = 0 + + def take_sample(self) -> Sample: + """Dispatch to container or native sampling, caching the resolved process.""" + if self.system in Containerize.supported_systems(): + return sample_container(self.system, self.container) + if self.worker_proc is None or not self.worker_proc.is_running(): + self.worker_proc = find_process_by_binary( + self.parent_pid, self.binary + ) + if self.worker_proc is None: + return Sample() + # cpu_percent reports usage since the previous call, so this + # first call seeds the baseline and its 0.0 result is discarded. + try: + self.worker_proc.cpu_percent(interval=None) + except (psutil.NoSuchProcess, psutil.AccessDenied): + self.worker_proc = None + return Sample() + return sample_process(self.worker_proc) + + def run_loop(self): + """ + Polling loop on a background thread. Samples resource usage + and appends one TSV row per iteration until stop_event is set. + """ + while not self.stop_event.is_set(): + sample = self.take_sample() + sample.elapsed_s = round(time.monotonic() - self.start_time, 1) + if sample.rss is not None and self.log_file is not None: + self.peak_rss = max(self.peak_rss, sample.rss) + self.log_file.write(sample_to_tsv_row(sample)) + self.log_file.flush() + self.stop_event.wait(self.interval) + + def __enter__(self): + """Open the TSV log, write the header, start sampling thread.""" + self.log_path = ( + self.output_dir / f"{self.dataset}.index.resource-usage-log.tsv" + ) + self.log_file = open(self.log_path, "w") + header = "\t".join(f.name for f in fields(Sample)) + "\n" + self.log_file.write(header) + self.log_file.flush() + self.start_time = time.monotonic() + self.thread = threading.Thread(target=self.run_loop, daemon=True) + self.thread.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Stop sampling, close the log, report where it was saved.""" + self.stop_event.set() + self.thread.join() + self.log_file.close() + if self.peak_rss > 0: + log.info( + "Resource-usage log (RSS memory and CPU usage) saved to " + f"`{self.log_path.name}`" + ) + else: + log.warning( + "Resource usage was not recorded (no samples collected)." + ) + return False diff --git a/src/qlever/resource_usage/usage_plot.py b/src/qlever/resource_usage/usage_plot.py index 9a447efc..a04bccfd 100644 --- a/src/qlever/resource_usage/usage_plot.py +++ b/src/qlever/resource_usage/usage_plot.py @@ -187,166 +187,179 @@ def build_plot_subtitle( return " | ".join(parts) if parts else None -def write_usage_plot( - tsv_path: Path, - log_path: Path, - stxxl_memory: str, - settings_json: str, - out_path: Path, - title: str, - plot_max_points: int = 500, -) -> bool: +class UsagePlot: """ - Read the usage TSV and index log, render a dual-axis plot of - memory and CPU over time with phase bands from the index log, - and save it to `out_path`. Returns True if a plot was saved, - False if the TSV has no usable samples. `plot_max_points` caps - the number of points drawn per series. + Render a resource-usage plot from an index build's TSV log. + Subclass and override `overlay` and/or `subtitle` for a specific + engine; the base versions describe a QLever index build. """ - data = read_usage_tsv(tsv_path) - if not data or len(data.get("elapsed_s", [])) == 0: - return False - - # drop leading rows where rss was never sampled so the plot starts - # at the first real measurement rather than a flat NaN run. - valid = np.where(~np.isnan(data["rss"]))[0] - if len(valid) == 0: - return False - data = {name: values[valid[0] :] for name, values in data.items()} - data["elapsed_s"] = data["elapsed_s"] - data["elapsed_s"][0] - - phases = compute_phase_boundaries(log_path) - - data = downsample_for_plot(data, plot_max_points) - elapsed_s = data["elapsed_s"] - - x_label, x_factor = pick_time_unit(float(elapsed_s[-1])) - x_values = elapsed_s / x_factor - rss_gb = data["rss"] / GB - cores = psutil.cpu_count() or 1 - # cpu_percent is per-core (100% == one fully used core), so dividing - # by 100 converts it to a count of cores for the CPU axis. - cpu_cores = data["cpu_percent"] / 100.0 - - fig, ax_mem = plt.subplots(figsize=(12, 6), constrained_layout=True) - ax_cpu = ax_mem.twinx() - - band_colors = plt.colormaps["Pastel1"].colors - total_s = float(elapsed_s[-1]) if len(elapsed_s) else 0.0 - # skip drawing the phase name when the band is too narrow to fit it - # legibly; arbitrary 2% of total duration. - min_label_s = total_s * 0.02 - for band_idx, (name, (start_s, end_s)) in enumerate(phases.items()): - band_s = end_s - start_s - if band_s <= 0: - continue - ax_mem.axvspan( - start_s / x_factor, - end_s / x_factor, - color=band_colors[band_idx % len(band_colors)], - alpha=0.4, - zorder=0, - ) - if band_s < min_label_s: - continue - mid = (start_s + end_s) / 2 / x_factor - ax_mem.text( - mid, - 0.98, - name, - transform=ax_mem.get_xaxis_transform(), - ha="center", - va="top", - rotation=90, - fontsize=8, - alpha=0.7, - ) - ax_mem.plot(x_values, rss_gb, color="#cc0000", label="RSS", linewidth=1.5) - ax_mem.set_xlabel(x_label) - ax_mem.set_ylabel("RSS Memory (GB)") - ax_mem.grid(True, linestyle="--", alpha=0.3) - - if not np.all(np.isnan(cpu_cores)): - ax_cpu.plot( - x_values, - cpu_cores, - color="#1f77b4", - label="CPU", - linewidth=1.2, - alpha=0.7, + def __init__( + self, + dataset: str, + args, + *, + output_dir: Path | None = None, + plot_max_points: int = 500, + ): + self.dataset = dataset + self.args = args + self.output_dir = output_dir or Path.cwd() + self.plot_max_points = plot_max_points + self.log_path = self.output_dir / f"{dataset}.index-log.txt" + + def overlay(self) -> list[tuple[str, float, float]]: + """Background regions as (label, start_s, end_s); empty if none.""" + phases = compute_phase_boundaries(self.log_path) + return [(name, start, end) for name, (start, end) in phases.items()] + + def subtitle(self) -> str | None: + """Subtitle line drawn under the title, or None.""" + return build_plot_subtitle( + self.log_path, + self.args.stxxl_memory or "", + self.args.settings_json, ) - ax_cpu.set_ylabel(f"CPU (cores, {cores} available)") - ax_cpu.set_ylim(0, cores) - - max_rss = float(np.nanmax(rss_gb)) - x_max = float(x_values[-1]) - # With a single sample both spans are zero, which makes the axis - # limits identical; fall back to a unit span so the limits differ. - y_span = max_rss if max_rss > 0 else 1.0 - x_span = x_max if x_max > 0 else 1.0 - ax_mem.set_ylim(-y_span * 0.04, y_span * 1.4) - ax_mem.set_xlim(-x_span * 0.02, x_span * 1.06) - - annotate_peak(ax_mem, x_values, rss_gb, "RSS", "#cc0000", (-25, 20)) - - lines_mem, labels_mem = ax_mem.get_legend_handles_labels() - lines_cpu, labels_cpu = ax_cpu.get_legend_handles_labels() - ax_mem.legend( - lines_mem + lines_cpu, - labels_mem + labels_cpu, - loc="center left", - bbox_to_anchor=(1.08, 0.5), - ) - - subtitle = build_plot_subtitle(log_path, stxxl_memory, settings_json) - ax_mem.set_title(f"{title}\n{subtitle}" if subtitle else title) - fig.savefig(out_path, dpi=120) - plt.close(fig) - return True - -def render_usage_plot( - dataset: str, - stxxl_memory: str = "", - settings_json: str = "{}", - output_dir: Path | None = None, - plot_max_points: int = 500, -) -> Path | None: - """ - Render `.resource-usage-plot.png` from - `.index.resource-usage-log.tsv` in `output_dir`, falling - back to `.resource-usage-log.tsv` as written by older - qlever versions. Returns the plot path on success, None if the log - is missing or the plot could not be rendered. - """ - output_dir = output_dir or Path.cwd() - tsv_path = output_dir / f"{dataset}.index.resource-usage-log.tsv" - # Backwards compatibility with older resource-usage log filename - if not tsv_path.exists(): - tsv_path = output_dir / f"{dataset}.resource-usage-log.tsv" - log_path = output_dir / f"{dataset}.index-log.txt" - plot_path = output_dir / f"{dataset}.resource-usage-plot.png" - if not tsv_path.exists(): - log.warning(f"Resource-usage log not found: `{tsv_path.name}`") - return None - try: - rendered = write_usage_plot( - tsv_path=tsv_path, - log_path=log_path, - stxxl_memory=stxxl_memory, - settings_json=settings_json, - out_path=plot_path, - title=f"{engine_name} index build: {dataset}", - plot_max_points=plot_max_points, + def build_figure(self, tsv_path: Path, plot_path: Path) -> bool: + """ + Read the usage TSV and save a dual-axis memory/CPU figure to + `plot_path`, shading `self.overlay()` regions and adding + `self.subtitle()` under the title. Returns True if a plot was + saved, False if the TSV has no usable samples. + """ + data = read_usage_tsv(tsv_path) + if not data or len(data.get("elapsed_s", [])) == 0: + return False + + # drop leading rows where rss was never sampled so the plot starts + # at the first real measurement rather than a flat NaN run. + valid = np.where(~np.isnan(data["rss"]))[0] + if len(valid) == 0: + return False + data = {name: values[valid[0] :] for name, values in data.items()} + data["elapsed_s"] = data["elapsed_s"] - data["elapsed_s"][0] + + overlay = self.overlay() + + data = downsample_for_plot(data, self.plot_max_points) + elapsed_s = data["elapsed_s"] + + x_label, x_factor = pick_time_unit(float(elapsed_s[-1])) + x_values = elapsed_s / x_factor + rss_gb = data["rss"] / GB + cores = psutil.cpu_count() or 1 + # cpu_percent is per-core (100% == one fully used core), so dividing + # by 100 converts it to a count of cores for the CPU axis. + cpu_cores = data["cpu_percent"] / 100.0 + + fig, ax_mem = plt.subplots(figsize=(12, 6), constrained_layout=True) + ax_cpu = ax_mem.twinx() + + band_colors = plt.colormaps["Pastel1"].colors + total_s = float(elapsed_s[-1]) if len(elapsed_s) else 0.0 + # skip drawing the region name when the band is too narrow to fit + # it legibly; arbitrary 2% of total duration. + min_label_s = total_s * 0.02 + for band_idx, (name, start_s, end_s) in enumerate(overlay): + band_s = end_s - start_s + if band_s <= 0: + continue + ax_mem.axvspan( + start_s / x_factor, + end_s / x_factor, + color=band_colors[band_idx % len(band_colors)], + alpha=0.4, + zorder=0, + ) + if band_s < min_label_s: + continue + mid = (start_s + end_s) / 2 / x_factor + ax_mem.text( + mid, + 0.98, + name, + transform=ax_mem.get_xaxis_transform(), + ha="center", + va="top", + rotation=90, + fontsize=8, + alpha=0.7, + ) + + ax_mem.plot( + x_values, rss_gb, color="#cc0000", label="RSS", linewidth=1.5 ) - except Exception as error: - log.warning(f"Could not render resource-usage plot: {error}") - return None - if not rendered: - log.warning( - "Resource-usage plot not rendered: no usable samples in the " - f"resource-usage log `{tsv_path.name}`" + ax_mem.set_xlabel(x_label) + ax_mem.set_ylabel("RSS Memory (GB)") + ax_mem.grid(True, linestyle="--", alpha=0.3) + + if not np.all(np.isnan(cpu_cores)): + ax_cpu.plot( + x_values, + cpu_cores, + color="#1f77b4", + label="CPU", + linewidth=1.2, + alpha=0.7, + ) + ax_cpu.set_ylabel(f"CPU (cores, {cores} available)") + ax_cpu.set_ylim(0, cores) + + max_rss = float(np.nanmax(rss_gb)) + x_max = float(x_values[-1]) + # With a single sample both spans are zero, which makes the axis + # limits identical; fall back to a unit span so the limits differ. + y_span = max_rss if max_rss > 0 else 1.0 + x_span = x_max if x_max > 0 else 1.0 + ax_mem.set_ylim(-y_span * 0.04, y_span * 1.4) + ax_mem.set_xlim(-x_span * 0.02, x_span * 1.06) + + annotate_peak(ax_mem, x_values, rss_gb, "RSS", "#cc0000", (-25, 20)) + + lines_mem, labels_mem = ax_mem.get_legend_handles_labels() + lines_cpu, labels_cpu = ax_cpu.get_legend_handles_labels() + ax_mem.legend( + lines_mem + lines_cpu, + labels_mem + labels_cpu, + loc="center left", + bbox_to_anchor=(1.08, 0.5), + ) + + title = f"{engine_name} index build: {self.dataset}" + subtitle = self.subtitle() + ax_mem.set_title(f"{title}\n{subtitle}" if subtitle else title) + fig.savefig(plot_path, dpi=120) + plt.close(fig) + return True + + def render(self) -> Path | None: + """ + Resolve the TSV and output paths, build the figure, and return + the saved plot path. Returns None if the log is missing or the + plot could not be rendered. + """ + tsv_path = ( + self.output_dir / f"{self.dataset}.index.resource-usage-log.tsv" ) - return None - return plot_path + # Backwards compatibility with older resource-usage log filename + if not tsv_path.exists(): + tsv_path = ( + self.output_dir / f"{self.dataset}.resource-usage-log.tsv" + ) + plot_path = self.output_dir / f"{self.dataset}.resource-usage-plot.png" + if not tsv_path.exists(): + log.warning(f"Resource-usage log not found: `{tsv_path.name}`") + return None + try: + rendered = self.build_figure(tsv_path, plot_path) + except Exception as error: + log.warning(f"Could not render resource-usage plot: {error}") + return None + if not rendered: + log.warning( + "Resource-usage plot not rendered: no usable samples in " + f"the resource-usage log `{tsv_path.name}`" + ) + return None + return plot_path diff --git a/src/qlever/util.py b/src/qlever/util.py index 0dfb2658..223bee66 100644 --- a/src/qlever/util.py +++ b/src/qlever/util.py @@ -508,6 +508,125 @@ def get_container_image_id(system: str, image: str) -> str: return image_id +def edit_option_line( + line: str, new_value: str, is_suffix: bool, comment_prefix: str | None +) -> str: + """ + Return `line` with its value replaced by `new_value`, or with + `new_value` appended to it if `is_suffix` is true. An inline + comment after the value is kept. + """ + # Split off an inline comment (whitespace followed by the comment + # prefix) so that only the value part is edited. + value_part = line + comment_part = "" + if comment_prefix is not None: + comment_match = re.search(rf"\s{re.escape(comment_prefix)}", line) + if comment_match: + value_part = line[: comment_match.start()] + comment_part = "\t" + line[comment_match.start() :].strip() + + if is_suffix: + new_line = value_part.rstrip() + new_value + else: + # Keep everything up to and including the `=` and the spacing + # after it, replace the old value. + prefix_end = re.match(r"^\s*\S+\s*=\s*", value_part).end() + new_line = value_part[:prefix_end] + new_value + return new_line + comment_part + + +def update_ini_values( + lines: list[str], + updates: dict[str, dict[str, tuple[str, bool]]], + comment_prefix: str | None = None, +) -> list[str]: + """ + Update values in INI-style file content given as `lines` and return + the modified lines, preserving comments and unrelated lines. + + `updates` maps `{section: {option: (new_value, is_suffix)}}`. An + existing option gets its value replaced, or `new_value` appended to + it if `is_suffix` is true. A missing option is added at the end of + its section, a missing section at the end of the file (suffix + entries are skipped there, they have no value to append to). + + `comment_prefix` is the inline comment character of the format + (`;` for `virtuoso.ini`). If None, the format has no inline + comments and the whole line is treated as the value. + """ + options_applied = {section: set() for section in updates} + sections_seen = set() + result_lines = [] + current_section = None + + def missing_option_lines(section: str) -> list[str]: + """ + Lines for options of `section` that were not found in the file. + """ + return [ + f"{option} = {value}" + for option, (value, is_suffix) in updates[section].items() + if option not in options_applied[section] and not is_suffix + ] + + def flush_missing_options(section: str): + """ + Insert options of `section` that were not found in the file, + before any blank lines that separate it from the next section. + """ + insert_at = len(result_lines) + while insert_at > 0 and result_lines[insert_at - 1].strip() == "": + insert_at -= 1 + result_lines[insert_at:insert_at] = missing_option_lines(section) + + for line in lines: + stripped = line.strip() + + # Section headers like `[Parameters]`. Commented-out headers + # like `;[Striping]` do not match because of the `^` anchor. + header_match = re.match(r"^\[([^\]]+)\]", stripped) + if header_match: + # Add options that were missing from the section we leave. + if current_section in updates: + flush_missing_options(current_section) + current_section = header_match.group(1) + sections_seen.add(current_section) + result_lines.append(line) + continue + + if current_section not in updates: + result_lines.append(line) + continue + + option_match = re.match(r"^(\S+)\s*=\s*", stripped) + if ( + option_match is None + or option_match.group(1) not in updates[current_section] + ): + result_lines.append(line) + continue + + option_name = option_match.group(1) + new_value, is_suffix = updates[current_section][option_name] + result_lines.append( + edit_option_line(line, new_value, is_suffix, comment_prefix) + ) + options_applied[current_section].add(option_name) + + # Add options missing from the last section in the file. + if current_section in updates: + flush_missing_options(current_section) + + # Add sections that were not in the file at all. + for section in updates: + if section not in sections_seen: + result_lines.append(f"\n[{section}]") + result_lines.extend(missing_option_lines(section)) + + return result_lines + + def get_ini_sed_cmd( section: str, option: str, new_value: str, is_suffix: bool = False ) -> str: diff --git a/src/qoxigraph/__init__.py b/src/qoxigraph/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qoxigraph/commands/__init__.py b/src/qoxigraph/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qoxigraph/commands/benchmark_queries.py b/src/qoxigraph/commands/benchmark_queries.py new file mode 100644 index 00000000..4f285520 --- /dev/null +++ b/src/qoxigraph/commands/benchmark_queries.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from qlever.commands.benchmark_queries import ( + BenchmarkQueriesCommand as QleverBenchmarkQueriesCommand, +) + + +class BenchmarkQueriesCommand(QleverBenchmarkQueriesCommand): + """ + Run benchmark queries against the Oxigraph SPARQL endpoint. + Overrides the default endpoint to use Oxigraph's /query path. + """ + + def execute(self, args) -> bool: + if not args.sparql_endpoint: + args.sparql_endpoint = f"{args.host_name}:{args.port}/query" + return super().execute(args) diff --git a/src/qoxigraph/commands/get_data.py b/src/qoxigraph/commands/get_data.py new file mode 100644 index 00000000..29bba0e2 --- /dev/null +++ b/src/qoxigraph/commands/get_data.py @@ -0,0 +1 @@ +from qlever.commands.get_data import GetDataCommand # noqa diff --git a/src/qoxigraph/commands/index.py b/src/qoxigraph/commands/index.py new file mode 100644 index 00000000..ef621db1 --- /dev/null +++ b/src/qoxigraph/commands/index.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import shlex +import time +from pathlib import Path + +import qlever.util as util +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log +from qlever.resource_usage.resource_monitor import ResourceMonitor + + +def wrap_cmd_in_container(args, cmd: str, ulimit: int | None = None) -> str: + """ + Wrap an indexing command in a container that is automatically removed + after the process exits (`--rm`) Use `use_bash=False` as Oxigraph image + doesn't support bash entrypoint. + """ + run_subcommand = "run --rm" + if ulimit: + run_subcommand += f" --ulimit nofile={ulimit}:{ulimit}" + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand=run_subcommand, + image_name=args.image, + container_name=args.index_container, + volumes=[("$(pwd)", "/opt")], + working_directory="/opt", + use_bash=False, + ) + + +def render_usage_plot(args, plot_only: bool = False) -> Path | None: + """Render the resource-usage plot. + + When the plotting libraries are missing, this is an error if the + user asked for the plot directly via `plot_only`, otherwise it notes + how to get the plot at info level since the index build succeeded. + """ + try: + from qoxigraph.resource_usage.usage_plot import UsagePlot + except ImportError: + if plot_only: + log.error( + "Resource-usage plot needs matplotlib and numpy " + "(`pip install qlever[plot]`). Install them and rerun." + ) + else: + log.info( + "To plot the resource-usage log, install matplotlib and " + "numpy (`pip install qlever[plot]`), then run " + "`qoxigraph index --resource-usage-plot-only`." + ) + return None + return UsagePlot( + args.name, args, plot_max_points=args.resource_usage_plot_max_points + ).render() + + +class IndexCommand(QleverCommand): + """ + Build an Oxigraph index for an RDF dataset. The indexing workflow is: + 1. Run `oxigraph load` to import input files into a RocksDB store. + 2. Optionally run `oxigraph optimize` to compact storage for read-only use. + + For large datasets (>5 GB), the file descriptor ulimit is raised + automatically because RocksDB opens many .sst files concurrently. + """ + + def __init__(self): + pass + + def description(self) -> str: + return "Build the index for a given RDF dataset" + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name", "format"], + "index": [ + "input_files", + "ulimit", + "index_binary", + "lenient", + "extra_args", + "resource_usage_interval", + "resource_usage_plot_max_points", + ], + "server": ["read_only"], + "runtime": ["system", "image", "index_container"], + } + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "--resource-usage-plot-only", + action="store_true", + default=False, + help="Only render the resource-usage plot from the existing " + "`.index.resource-usage-log.tsv`; do not build the index. " + "Use after installing the plotting libraries, or to re-render " + "with a different `--resource-usage-plot-max-points`", + ) + + def execute(self, args) -> bool: + # Render the resource-usage plot from the existing log without + # rebuilding the index. + if args.resource_usage_plot_only: + plot_path = render_usage_plot(args, plot_only=True) + if plot_path is None: + return False + log.info(f"Resource-usage plot saved to `{plot_path.name}`") + return True + + cmds_to_execute = [] + index_cmd = ( + f"load {'--lenient ' if args.lenient == 'yes' else ''}" + f"--location {args.name}_index/ --file {args.input_files} " + f"{args.extra_args} |& tee {args.name}.index-log.txt" + ) + + ulimit = args.ulimit + # RocksDB opens many .sst files concurrently. For datasets larger + # than 5 GB, raise the file descriptor limit so the process does + # not hit the default OS soft limit. + total_file_size = util.get_total_file_size( + shlex.split(args.input_files) + ) + if not ulimit and total_file_size > 5e9: + ulimit = 500_000 + if args.system in Containerize.supported_systems(): + index_cmd = wrap_cmd_in_container(args, index_cmd, ulimit) + else: + index_cmd = f"{args.index_binary} {index_cmd}" + if ulimit: + index_cmd = f"ulimit -Sn {ulimit} && {index_cmd}" + + cmds_to_execute.append(index_cmd) + + # Compact the RocksDB storage for read-only serving. This reduces + # disk usage and speeds up queries but makes the index immutable. + optimize_cmd = None + if args.read_only == "yes": + optimize_cmd = f"optimize -l {args.name}_index/" + if args.system in Containerize.supported_systems(): + optimize_cmd = wrap_cmd_in_container(args, optimize_cmd) + else: + optimize_cmd = f"{args.index_binary} {optimize_cmd}" + cmds_to_execute.append(optimize_cmd) + + # Show the command line. + self.show("\n".join(cmds_to_execute), only_show=args.show) + if args.show: + return True + + if not util.input_files_exist(args.input_files): + return False + + # When running natively, check if the binary exists and works. + if args.system in Containerize.supported_systems(): + if Containerize().is_running(args.system, args.index_container): + log.info( + f"{args.system} container {args.index_container} is still up, " + "which means that data loading is in progress. Please wait..." + ) + return False + else: + if not util.binary_exists(args.index_binary, "index-binary", args): + return False + + # Abort if a previous index already exists. RocksDB .sst files in + # the index directory indicate an existing store. + if ( + len([p.name for p in Path(f"{args.name}_index").glob("*.sst")]) + != 0 + ): + log.error( + f"Index files (*.sst) found in {args.name}_index directory " + "which shows presence of a previous index" + ) + log.info("") + log.info("Aborting the index operation...") + return False + + # Run the index command and record the elapsed time in the log + # file. Oxigraph's progress output is unreliable (may not print a + # final summary line when loading multiple files), so we measure + # the time externally. + # + log_file_name = f"{args.name}.index-log.txt" + with ResourceMonitor( + dataset=args.name, + binary=args.index_binary, + container=args.index_container, + system=args.system, + interval=args.resource_usage_interval, + ): + try: + load_start = time.time() + util.run_command(index_cmd, show_output=True, show_stderr=True) + load_s = time.time() - load_start + except Exception as e: + log.error(f"Building the index failed: {e}") + return False + + optimize_s = 0.0 + if optimize_cmd: + try: + log.info("") + log.info("Optimizing read-only database storage:") + self.show(optimize_cmd) + optimize_start = time.time() + util.run_command( + optimize_cmd, show_output=True, show_stderr=True + ) + optimize_s = time.time() - optimize_start + except Exception as e: + log.error(f"Optimizing the database storage failed: {e}") + log.info( + f"Please run manually: " + f"{args.index_binary} optimize -l {args.name}_index/" + ) + + with open(log_file_name, "a") as f: + f.write(f"Load time: {load_s:.0f}s\n") + if optimize_cmd: + f.write(f"Optimize time: {optimize_s:.0f}s\n") + f.write(f"TOTAL time: {load_s + optimize_s:.0f}s\n") + + plot_path = render_usage_plot(args) + if plot_path is not None: + log.info(f"Resource-usage plot saved to `{plot_path.name}`") + + return True diff --git a/src/qoxigraph/commands/index_stats.py b/src/qoxigraph/commands/index_stats.py new file mode 100644 index 00000000..bdfdbe39 --- /dev/null +++ b/src/qoxigraph/commands/index_stats.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import qlever.util as util +from qlever.commands.index_stats import ( + IndexStatsCommand as QleverIndexStatsCommand, +) +from qlever.commands.index_stats import ( + get_size_unit, + get_size_unit_factor, + get_time_unit, + get_time_unit_factor, +) +from qlever.log import log +from qoxigraph.resource_usage.usage_plot import parse_logged_seconds + + +class IndexStatsCommand(QleverIndexStatsCommand): + """ + Show index build time and disk space usage for an Oxigraph dataset. + Time is read from the "TOTAL time" line appended to the index log + by the index command; space is the sum of all .sst files. + """ + + def execute_time( + self, args, log_file_name: str + ) -> dict[str, tuple[float | None, str]]: + """Parse index build times from the index log file.""" + try: + # Read the last few lines of the log file (the times are + # always near the end). + log_text = util.run_command( + f"tail {log_file_name}", return_output=True + ) + except Exception as e: + log.error(f"Problem reading index log file {log_file_name}: {e}") + return {} + + phases = ["Load time", "Optimize time", "TOTAL time"] + + raw_seconds = {} + for name in phases: + seconds = parse_logged_seconds(log_text, f"{name}:") + if seconds is not None: + raw_seconds[name] = seconds + + if not raw_seconds: + return {} + + # Pick a time unit based on the total time. + total_s = raw_seconds.get("TOTAL time") + time_unit = get_time_unit(args.time_unit, total_s) + unit_factor = get_time_unit_factor(time_unit) + + stats = {} + for name in phases: + if name in raw_seconds: + stats[name] = (raw_seconds[name] / unit_factor, time_unit) + + # If there was no optimize step, Load and TOTAL are identical + if "Optimize time" not in stats: + stats.pop("Load time", None) + + return stats + + def execute_space(self, args) -> dict[str, tuple[float, str]]: + """ + Return the space used by the index files (*.sst) along with the unit. + """ + index_size = util.get_total_file_size([f"{args.name}_index/*.sst"]) + + size_unit = get_size_unit(args.size_unit, index_size) + unit_factor = get_size_unit_factor(size_unit) + + index_size /= unit_factor + + return {"TOTAL size": (index_size, size_unit)} diff --git a/src/qoxigraph/commands/log.py b/src/qoxigraph/commands/log.py new file mode 100644 index 00000000..401d2148 --- /dev/null +++ b/src/qoxigraph/commands/log.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from qlever import script_name +from qlever.commands.log import LogCommand as QleverLogCommand +from qlever.containerize import Containerize +from qlever.log import log +from qlever.util import run_command + + +class LogCommand(QleverLogCommand): + """ + Show server logs for Oxigraph. For native execution, tails the log + file as usual. For containers, uses `docker/podman logs` as it is + not possible to redirect oxigraph logs to a log file. + """ + + def __init__(self): + pass + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "runtime": [ + "system", + "image", + "server_container", + ], + } + + def execute(self, args) -> bool: + if args.system not in Containerize.supported_systems(): + return super().execute(args) + + # Handle container logging using docker/podman logs command instead of tail + # This is because we don't have .server-log.txt for + # containerized execution + log_cmd = f"{args.system} logs " + + if not args.from_beginning: + log_cmd += f"-n {args.tail_num_lines} " + if not args.no_follow: + log_cmd += "-f " + + log_cmd += args.server_container + + # Show the command line. + self.show(log_cmd, only_show=args.show) + if args.show: + return True + + if not Containerize().is_running(args.system, args.server_container): + log.error(f"No server container {args.server_container} found!\n") + log.info(f"Are you sure you called `{script_name} start`?") + return False + + try: + run_command(log_cmd, show_output=True, show_stderr=True) + except Exception as e: + log.error(f"Cannot display container logs - {e}") + return True diff --git a/src/qoxigraph/commands/query.py b/src/qoxigraph/commands/query.py new file mode 100644 index 00000000..bc3fb35c --- /dev/null +++ b/src/qoxigraph/commands/query.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from qlever.commands.query import QueryCommand as QleverQueryCommand + + +class QueryCommand(QleverQueryCommand): + """ + Send a SPARQL query to the Oxigraph server. Extends the base query + command with Oxigraph's /query endpoint and supported result formats. + This class is used as the base QueryCommand by all the other new engines. + """ + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "query", + type=str, + nargs="?", + default="SELECT * WHERE { ?s ?p ?o } LIMIT 10", + help="SPARQL query to send", + ) + subparser.add_argument( + "--predefined-query", + type=str, + choices=self.predefined_queries.keys(), + help="Use a predefined query", + ) + subparser.add_argument( + "--sparql-endpoint", type=str, help="URL of the SPARQL endpoint" + ) + subparser.add_argument( + "--accept", + type=str, + choices=[ + "text/tab-separated-values", + "text/csv", + "application/sparql-results+json", + "application/sparql-results+xml", + ], + default="text/tab-separated-values", + help="Accept header for the SPARQL query", + ) + subparser.add_argument( + "--get", + action="store_true", + default=False, + help="Use GET request instead of POST", + ) + subparser.add_argument( + "--no-time", + action="store_true", + default=False, + help="Do not print the (end-to-end) time taken", + ) + + def execute(self, args) -> bool: + # Oxigraph's SPARQL endpoint is at /query. + if not args.sparql_endpoint: + args.sparql_endpoint = f"{args.host_name}:{args.port}/query" + # These QLever-specific options are not supported by Oxigraph. + args.pin_to_cache = None + args.access_token = None + return super().execute(args) diff --git a/src/qoxigraph/commands/setup_config.py b/src/qoxigraph/commands/setup_config.py new file mode 100644 index 00000000..ed286269 --- /dev/null +++ b/src/qoxigraph/commands/setup_config.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from configparser import RawConfigParser +from pathlib import Path + +from qlever.commands.setup_config import ( + SetupConfigCommand as QleverSetupConfigCommand, +) +from qlever.log import log +from qlever.qleverfile import Qleverfile + + +class SetupConfigCommand(QleverSetupConfigCommand): + """ + Create a Qleverfile for Oxigraph from a dataset template from `src/qlever/Qleverfiles`. + Filters the template to keep only the relevant sections and adds Oxigraph-specific + defaults (read-only mode, query timeout). + This class is used as the base SetupConfigCommand by all the other new engines. + """ + + IMAGE = "ghcr.io/oxigraph/oxigraph" + + # Sections and keys to retain when filtering a Qleverfile template. + FILTER_CRITERIA = { + "data": [], + "index": ["INPUT_FILES"], + "server": ["PORT"], + "runtime": ["SYSTEM", "IMAGE"], + "ui": ["UI_CONFIG"], + } + + @staticmethod + def construct_engine_specific_params(args) -> dict[str, dict[str, str]]: + """Return Oxigraph-specific defaults to inject into the Qleverfile.""" + return {"server": {"READ_ONLY": "yes", "TIMEOUT": "60s"}} + + @staticmethod + def add_engine_specific_option_values( + qleverfile_parser: RawConfigParser, + engine_specific_params: dict[str, dict[str, str]], + ) -> None: + """Merge engine-specific parameters into the Qleverfile parser.""" + for section, option_dict in engine_specific_params.items(): + if qleverfile_parser.has_section(section): + for option, value in option_dict.items(): + qleverfile_parser.set(section, option, value) + + def execute(self, args) -> bool: + # Construct the command line and show it. + template_path = ( + self.qleverfiles_path / f"Qleverfile.{args.config_name}" + ) + setup_config_show = ( + f"Qleverfile for {args.config_name} will be created using " + f"Qleverfile.{args.config_name} file in {template_path}" + ) + self.show(setup_config_show, only_show=args.show) + if args.show: + return True + + # If there is already a Qleverfile in the current directory, exit. + if self.check_qleverfile_exists(): + return False + + qleverfile_path = Path("Qleverfile") + + try: + qleverfile_parser = Qleverfile.filter( + template_path, self.FILTER_CRITERIA + ) + qleverfile_parser.set("runtime", "IMAGE", self.IMAGE) + params = self.construct_engine_specific_params(args) + self.add_engine_specific_option_values(qleverfile_parser, params) + for section, arg_name in self.override_args: + if arg_value := getattr(args, arg_name, None): + qleverfile_parser.set( + section, arg_name.upper(), str(arg_value) + ) + with qleverfile_path.open("w") as f: + qleverfile_parser.write(f) + + log.info( + f'Created Qleverfile for config "{args.config_name}"' + f" in current directory" + ) + return True + except Exception as e: + log.error( + f'Could not copy "{qleverfile_path}" to current directory: {e}' + ) + return False diff --git a/src/qoxigraph/commands/start.py b/src/qoxigraph/commands/start.py new file mode 100644 index 00000000..dd049a70 --- /dev/null +++ b/src/qoxigraph/commands/start.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import subprocess +import time +from pathlib import Path + +from qlever import script_name +from qlever.command import QleverCommand +from qlever.containerize import Containerize +from qlever.log import log +from qlever.util import ( + binary_exists, + is_server_alive, + run_command, + tail_log_file, +) +from qoxigraph.commands.stop import StopCommand + + +def timeout_supported(args, serve_ps: str) -> bool: + """Check whether the oxigraph server binary supports query timeouts.""" + help_cmd = f"{serve_ps} --help" + if args.system in Containerize.supported_systems(): + help_cmd = f"{args.system} run --rm {args.image} {help_cmd}" + else: + help_cmd = f"{args.server_binary} {help_cmd}" + try: + help_output = run_command(help_cmd, return_output=True) + return "timeout-s" in help_output + except Exception as e: + log.warning( + "Could not determine if query timeouts are supported by this version " + f"of Oxigraph! Falling back to no timeouts. Error: {e}", + ) + return False + + +def wrap_cmd_in_container(args, cmd: str) -> str: + """Wrap the server start command in a container with restart policy.""" + run_subcommand = "run --restart=unless-stopped" + if not args.run_in_foreground: + run_subcommand += " -d" + return Containerize().containerize_command( + cmd=cmd, + container_system=args.system, + run_subcommand=run_subcommand, + image_name=args.image, + container_name=args.server_container, + volumes=[("$(pwd)", "/opt")], + ports=[(args.port, args.port)], + working_directory="/opt", + use_bash=False, + ) + + +class StartCommand(QleverCommand): + """ + Start the Oxigraph SPARQL server for an already-indexed dataset. + Supports both native and containerized execution, with an option + to run in the foreground. Uses `serve-read-only` or `serve` + depending on the read_only setting. + """ + + def __init__(self): + pass + + def description(self) -> str: + return ( + "Start the server for Oxigraph (requires that you have built an " + "index before)" + ) + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "server": [ + "host_name", + "port", + "read_only", + "server_binary", + "timeout", + "extra_args", + ], + "runtime": ["system", "image", "server_container"], + } + + def additional_arguments(self, subparser): + subparser.add_argument( + "--run-in-foreground", + action="store_true", + default=False, + help=( + "Run the start command in the foreground " + "(default: run in the background)" + ), + ) + + def execute(self, args) -> bool: + # Inside a container, bind to 0.0.0.0 so the port mapping is + # reachable from the host; natively, bind to the configured host. + bind = ( + f"0.0.0.0:{args.port}" + if args.system in Containerize.supported_systems() + else f"{args.host_name}:{args.port}" + ) + process = "serve-read-only" if args.read_only == "yes" else "serve" + timeout_str = "" + if timeout_supported(args, process): + try: + timeout_s = int(args.timeout[:-1]) + except ValueError as e: + log.warning( + f"Invalid timeout value {args.timeout}. Error: {e}" + ) + log.info("Setting timeout to 60s!") + timeout_s = 60 + timeout_str = f"--timeout-s {timeout_s}" + else: + log.info( + f"Ignoring the set timeout value of {args.timeout} as your " + "version of Oxigraph doesn't currently support query timeouts!" + ) + + start_cmd = ( + f"{process} --location {args.name}_index/ {args.extra_args} " + f"{timeout_str} --bind={bind}" + ) + + if args.system in Containerize.supported_systems(): + start_cmd = wrap_cmd_in_container(args, start_cmd) + else: + start_cmd = f"{args.server_binary} {start_cmd} > {args.name}.server-log.txt 2>&1" + if not args.run_in_foreground: + start_cmd = f"nohup {start_cmd} &" + + # Show the command line. + self.show(start_cmd, only_show=args.show) + if args.show: + return True + + endpoint_url = f"http://{args.host_name}:{args.port}/query" + + # When running natively, check if the binary exists and works. + if args.system not in Containerize.supported_systems(): + if not binary_exists(args.server_binary, "server-binary", args): + return False + + # Check if index files (*.sst) present in index directory + if ( + len([p.name for p in Path(f"{args.name}_index/").glob("*.sst")]) + == 0 + ): + log.error(f"No Oxigraph index files for {args.name} found!\n") + log.info( + f"Did you call `{script_name} index`? If you did, check " + "if .sst index files are present in index directory." + ) + return False + + # Check if server already alive at endpoint url from a previous run + if is_server_alive(url=endpoint_url): + log.error(f"Oxigraph server already running on {endpoint_url}\n") + log.info(f"To kill the existing server, use `{script_name} stop`") + return False + + # Remove old log file so that tail starts clean. + log_file = Path(f"{args.name}.server-log.txt") + log_file.unlink(missing_ok=True) + + try: + process = run_command( + start_cmd, + use_popen=args.run_in_foreground, + ) + except Exception as e: + log.error(f"Starting the Oxigraph server failed ({e})") + return False + + # Tail the server log until the server is ready (note that the `exec` + # is important to make sure that the tail process is killed and not + # just the bash process). + if args.run_in_foreground: + log.info( + "Follow the server logs as long as the server is" + " running (Ctrl-C stops the server)" + ) + else: + log.info( + "Follow the server logs until the server is ready" + " (Ctrl-C stops following the log, but NOT the server)" + ) + log.info("") + # For containers, use `docker/podman logs -f` as Oxigraph doesn't + # support redirecting logs to a log file. A short delay ensures + # the container is up before attaching. + if args.system in Containerize.supported_systems(): + time.sleep(2) + log_cmd = f"exec {args.system} logs -f {args.server_container}" + log_proc = subprocess.Popen(log_cmd, shell=True) + else: + log_proc = tail_log_file(log_file) + if log_proc is None: + return False + while not is_server_alive(endpoint_url): + time.sleep(1) + + log.info( + f"Oxigraph server webapp for {args.name} will be available at " + f"http://{args.host_name}:{args.port} and the sparql endpoint for " + f"queries is {endpoint_url} when the server is ready" + ) + + # Kill the log process + if not args.run_in_foreground: + log_proc.terminate() + + # With `--run-in-foreground`, wait until the server is stopped. + # On Ctrl-C, terminate the process and clean up the container. + if args.run_in_foreground: + try: + process.wait() + except KeyboardInterrupt: + process.terminate() + if args.system in Containerize.supported_systems(): + args.cmdline_regex = StopCommand.DEFAULT_REGEX + StopCommand().execute(args) + log_proc.terminate() + + return True diff --git a/src/qoxigraph/commands/status.py b/src/qoxigraph/commands/status.py new file mode 100644 index 00000000..d73548dc --- /dev/null +++ b/src/qoxigraph/commands/status.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from qlever.commands.status import StatusCommand as QleverStatusCommand + + +class StatusCommand(QleverStatusCommand): + """Show Oxigraph server processes running on this machine.""" + + DEFAULT_REGEX = "oxigraph\\s+serve" + + def description(self) -> str: + return "Show Oxigraph processes running on this machine" + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "--cmdline-regex", + default=self.DEFAULT_REGEX, + help=( + "Show only processes where the command line matches this regex" + ), + ) diff --git a/src/qoxigraph/commands/stop.py b/src/qoxigraph/commands/stop.py new file mode 100644 index 00000000..47308284 --- /dev/null +++ b/src/qoxigraph/commands/stop.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from qlever.command import QleverCommand +from qlever.commands import stop as qlever_stop +from qlever.containerize import Containerize +from qlever.log import log +from qlever.util import stop_process_with_regex +from qoxigraph.commands.status import StatusCommand + + +class StopCommand(QleverCommand): + """ + Stop the Oxigraph server for a given dataset. For native execution, + finds and kills processes matching the dataset-name regex. For + containers, stops and removes the server container. + """ + + # Override this with StatusCommand from child class for execute + # method to work as intended + STATUS_COMMAND = StatusCommand() + # %%NAME%% is replaced at runtime with the dataset name from the Qleverfile + DEFAULT_REGEX = "oxigraph\\s+serve.*%%NAME%%_index" + + def __init__(self): + pass + + def description(self) -> str: + return "Stop Oxigraph server for a given dataset" + + def should_have_qleverfile(self) -> bool: + return True + + def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: + return { + "data": ["name"], + "runtime": ["system", "server_container"], + } + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "--cmdline-regex", + default=self.DEFAULT_REGEX, + help="Show only processes where the command " + "line matches this regex", + ) + + def execute(self, args) -> bool: + # Substitute the dataset name into the regex template so we only + # match the server running for this dataset. + cmdline_regex = args.cmdline_regex + if "%%NAME%%" in args.cmdline_regex and hasattr(args, "name"): + cmdline_regex = args.cmdline_regex.replace( + "%%NAME%%", str(args.name) + ) + description = ( + f"Checking for container with name {args.server_container}" + if args.system in Containerize.supported_systems() + else f'Checking for processes matching "{cmdline_regex}"' + ) + + self.show(description, only_show=args.show) + if args.show: + return True + + if args.system not in Containerize.supported_systems(): + stop_process_results = stop_process_with_regex(cmdline_regex) + if stop_process_results is None: + return False + if len(stop_process_results) > 0: + return all(stop_process_results) + + # If no matching process found, show a message and the output of the + # status command. + log.error("No matching process found") + args.cmdline_regex = self.STATUS_COMMAND.DEFAULT_REGEX + log.info("") + StatusCommand().execute(args) + return True + + # First check if container is running and if yes, stop and remove it + return qlever_stop.stop_container(args.server_container) diff --git a/src/qoxigraph/qleverfile.py b/src/qoxigraph/qleverfile.py new file mode 100644 index 00000000..467b77fa --- /dev/null +++ b/src/qoxigraph/qleverfile.py @@ -0,0 +1,77 @@ +from __future__ import annotations + + +def qleverfile_args(all_args: dict[str, dict[str, tuple]]) -> None: + """Define additional oxigraph specific Qleverfile parameters""" + + def arg(*args, **kwargs): + return (args, kwargs) + + index_args = all_args["index"] + server_args = all_args["server"] + + index_args["index_binary"] = arg( + "--index-binary", + type=str, + default="oxigraph", + help=( + "The binary for building the index (default: oxigraph) " + "(this requires that you have oxigraph-cli installed " + "on your machine)" + ), + ) + index_args["lenient"] = arg( + "--lenient", + type=str, + choices=["yes", "no"], + default="no", + help="Attempt to keep loading even if the data file is invalid", + ) + index_args["extra_args"] = arg( + "--extra-args", + type=str, + default="", + help=( + "Additional arguments to pass directly to the oxigraph load process. " + "This allows advanced users to specify options not exposed in " + "Qleverfile. The string is appended verbatim to the command." + ), + ) + + server_args["server_binary"] = arg( + "--server-binary", + type=str, + default="oxigraph", + help=( + "The binary for starting the server (default: oxigraph) " + "(this requires that you have oxigraph-cli installed " + "on your machine)" + ), + ) + server_args["read_only"] = arg( + "--read-only", + type=str, + choices=["yes", "no"], + default="yes", + help=( + "The HTTP server will not permit mutation operations in " + "read-only mode" + ), + ) + server_args["timeout"] = arg( + "--timeout", + type=str, + default="60s", + help="The maximal time in seconds a query is allowed to run", + ) + server_args["extra_args"] = arg( + "--extra-args", + type=str, + default="", + help=( + "Additional arguments to pass directly to the oxigraph " + "serve/serve-read-only. This allows advanced users to specify " + "options not exposed in Qleverfile. The string is appended " + "verbatim to the command." + ), + ) diff --git a/src/qoxigraph/resource_usage/__init__.py b/src/qoxigraph/resource_usage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/qoxigraph/resource_usage/usage_plot.py b/src/qoxigraph/resource_usage/usage_plot.py new file mode 100644 index 00000000..cfd3ae2d --- /dev/null +++ b/src/qoxigraph/resource_usage/usage_plot.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import re + +from qlever.containerize import Containerize +from qlever.resource_usage.usage_plot import UsagePlot as BaseUsagePlot +from qlever.util import run_command + + +def parse_logged_seconds(text: str, label: str) -> float | None: + """Return the integer seconds logged after `label`, or None.""" + match = re.search(rf"{re.escape(label)}\s*(\d+)s", text) + return float(match.group(1)) if match else None + + +class UsagePlot(BaseUsagePlot): + """Resource-usage plot for an Oxigraph index build.""" + + def overlay(self) -> list[tuple[str, float, float]]: + """Shade the load and optimize phases; empty if no optimize step.""" + try: + text = self.log_path.read_text() + except OSError: + return [] + load_s = parse_logged_seconds(text, "Load time:") + optimize_s = parse_logged_seconds(text, "Optimize time:") + if load_s is None or optimize_s is None: + return [] + return [ + ("Load", 0.0, load_s), + ("Optimize", load_s, load_s + optimize_s), + ] + + def subtitle(self) -> str | None: + """Assemble a 'version | read-only' line from the index args.""" + if self.args.system in Containerize.supported_systems(): + version_cmd = ( + f"{self.args.system} run --rm {self.args.image} --version" + ) + else: + version_cmd = f"{self.args.index_binary} --version" + try: + version = run_command(version_cmd, return_output=True).strip() + except Exception: + version = "" + parts = [] + if version: + parts.append(version) + parts.append(f"read-only = {self.args.read_only}") + return " | ".join(parts) diff --git a/test/qlever/resource_usage/test_resource_monitor.py b/test/qlever/resource_usage/test_resource_monitor.py new file mode 100644 index 00000000..ad94409f --- /dev/null +++ b/test/qlever/resource_usage/test_resource_monitor.py @@ -0,0 +1,59 @@ +from unittest.mock import MagicMock + +import psutil +import pytest + +from qlever.resource_usage.resource_monitor import ( + Sample, + sample_container, + sample_process, + sample_to_tsv_row, +) + +MODULE = "qlever.resource_usage.resource_monitor" + + +@pytest.mark.parametrize( + "sample,expected", + [ + (Sample(elapsed_s=1.0, rss=100, cpu_percent=5.0), "1.0\t100\t5.0\n"), + (Sample(), "\t\t\n"), + (Sample(elapsed_s=2.0), "2.0\t\t\n"), + # Zero is a real reading, not a missing one: it renders as "0" + # / "0.0", never as an empty column. + (Sample(elapsed_s=0.0, rss=0, cpu_percent=0.0), "0.0\t0\t0.0\n"), + ], +) +def test_sample_to_tsv_row(sample, expected): + assert sample_to_tsv_row(sample) == expected + + +def test_sample_container_parses_stats_output(mock_command): + run_cmd_mock = mock_command(MODULE, "run_command") + run_cmd_mock.return_value = "1.5GiB / 7.6GiB\t12.5%" + sample = sample_container("docker", "qlever.index.test") + assert sample.rss == int(1.5 * 1024**3) + assert sample.cpu_percent == 12.5 + + +def test_sample_container_returns_empty_on_malformed_output(mock_command): + run_cmd_mock = mock_command(MODULE, "run_command") + run_cmd_mock.return_value = "garbage" + sample = sample_container("docker", "qlever.index.test") + assert sample == Sample() + + +def test_sample_process_reads_rss_and_cpu(): + proc = MagicMock() + proc.memory_info.return_value.rss = 2048 + proc.cpu_percent.return_value = 7.5 + sample = sample_process(proc) + assert sample.rss == 2048 + assert sample.cpu_percent == 7.5 + + +def test_sample_process_returns_empty_when_process_gone(): + proc = MagicMock() + proc.memory_info.side_effect = psutil.NoSuchProcess(pid=123) + sample = sample_process(proc) + assert sample == Sample() diff --git a/test/qlever/resource_usage/test_usage_plot.py b/test/qlever/resource_usage/test_usage_plot.py index c0efa55e..e7141d1f 100644 --- a/test/qlever/resource_usage/test_usage_plot.py +++ b/test/qlever/resource_usage/test_usage_plot.py @@ -1,3 +1,5 @@ +from types import SimpleNamespace + import pytest # The plot extra (numpy, matplotlib) is optional, so skip this whole @@ -6,12 +8,12 @@ pytest.importorskip("matplotlib") from qlever.resource_usage.usage_plot import ( # noqa: E402 + UsagePlot, build_plot_subtitle, compute_phase_boundaries, downsample_for_plot, pick_time_unit, read_usage_tsv, - render_usage_plot, ) @@ -210,7 +212,7 @@ def test_compute_phase_boundaries_skips_incomplete_phase(tmp_path): def test_render_usage_plot_missing_tsv(tmp_path): - assert render_usage_plot("missing", output_dir=tmp_path) is None + assert UsagePlot("missing", None, output_dir=tmp_path).render() is None def test_render_usage_plot_header_only_tsv_renders_nothing(tmp_path): @@ -218,7 +220,7 @@ def test_render_usage_plot_header_only_tsv_renders_nothing(tmp_path): # or leave a PNG behind. tsv_path = tmp_path / "data.index.resource-usage-log.tsv" tsv_path.write_text("elapsed_s\trss\tcpu_percent\n") - assert render_usage_plot("data", output_dir=tmp_path) is None + assert UsagePlot("data", None, output_dir=tmp_path).render() is None assert not (tmp_path / "data.resource-usage-plot.png").exists() @@ -228,6 +230,7 @@ def test_render_usage_plot_falls_back_to_old_tsv_name(tmp_path): tsv_path.write_text( "elapsed_s\trss\tcpu_percent\n1.0\t100\t5.0\n2.0\t200\t6.0\n" ) - plot_path = render_usage_plot("data", output_dir=tmp_path) + args = SimpleNamespace(stxxl_memory="", settings_json="{}") + plot_path = UsagePlot("data", args, output_dir=tmp_path).render() assert plot_path == tmp_path / "data.resource-usage-plot.png" assert plot_path.exists()