From 7a4957ad1308a821ba2f42590432a83d331e4175 Mon Sep 17 00:00:00 2001 From: tanmay-9 Date: Sat, 15 Aug 2026 15:51:56 +0200 Subject: [PATCH 1/7] Make qlever commands engine-agnostic and clean up style issues --- src/qlever/command.py | 14 ++--- src/qlever/commands/add_text_index.py | 2 +- src/qlever/commands/benchmark_queries.py | 33 ++++++----- src/qlever/commands/index.py | 16 +++++- src/qlever/commands/index_stats.py | 34 +++++------ src/qlever/commands/settings.py | 2 +- src/qlever/commands/setup_config.py | 8 +-- src/qlever/commands/start.py | 8 +-- src/qlever/commands/system_info.py | 7 ++- src/qlever/commands/ui.py | 39 ++----------- src/qlever/commands/update_wikidata.py | 34 +++++------ src/qlever/config.py | 14 ++++- src/qlever/qlever_main.py | 2 +- src/qlever/qleverfile.py | 57 +++++++++---------- src/qlever/resource_usage/usage_plot.py | 4 +- src/qlever/util.py | 29 ++++++++-- .../test_benchmark_queries_methods.py | 2 +- test/qlever/commands/test_index_execute.py | 1 + .../commands/test_start_other_methods.py | 4 +- test/qlever/resource_usage/test_usage_plot.py | 6 +- 20 files changed, 165 insertions(+), 151 deletions(-) diff --git a/src/qlever/command.py b/src/qlever/command.py index a4d95923..573b28e2 100644 --- a/src/qlever/command.py +++ b/src/qlever/command.py @@ -9,7 +9,7 @@ class QleverCommand(ABC): """ - Abstract base class for all the commands in `qlever/commands`. + Abstract base class for all the commands in `/commands`. """ @abstractmethod @@ -21,15 +21,13 @@ def __init__(self): assignments, if any) because we create one object per command and initialize each of them. """ - pass @abstractmethod def description(self) -> str: """ A concise description of the command, which will be shown when the user - types `qlever --help` or `qlever --help`. + asks for the help of the engine or of the command itself. """ - pass @abstractmethod def should_have_qleverfile(self) -> bool: @@ -39,26 +37,23 @@ def should_have_qleverfile(self) -> bool: specified, the command can still be executed if all the required arguments are specified on the command line, but there will be warning. """ - pass @abstractmethod def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: """ Retun the arguments relevant for this command. This must be a subset of - the names of `all_arguments` defined in `QleverConfig`. Only these + the names of `all_arguments` defined in `Qleverfile`. Only these arguments can then be used in the `execute` method. """ - pass @abstractmethod def additional_arguments(self, subparser): """ Add additional command-specific arguments (which are not in - `QleverConfig.all_arguments` and cannot be specified in the Qleverfile) + `Qleverfile.all_arguments` and cannot be specified in the Qleverfile) to the given `subparser`. If there are no additional arguments, just implement as `pass`. """ - pass @abstractmethod def execute(self, args) -> bool: @@ -68,7 +63,6 @@ def execute(self, args) -> bool: the problem could be identified and handled. In all other cases, raise a `CommandException`. """ - pass @staticmethod def show(command_description: str, only_show: bool = False): diff --git a/src/qlever/commands/add_text_index.py b/src/qlever/commands/add_text_index.py index 0007e94e..3aa883bc 100644 --- a/src/qlever/commands/add_text_index.py +++ b/src/qlever/commands/add_text_index.py @@ -17,7 +17,7 @@ def __init__(self): pass def description(self) -> str: - return "Add text index to an index built with `qlever index`" + return "Add text index to an index built with the `index` command" def should_have_qleverfile(self) -> bool: return True diff --git a/src/qlever/commands/benchmark_queries.py b/src/qlever/commands/benchmark_queries.py index 737be31b..5ac44b4f 100644 --- a/src/qlever/commands/benchmark_queries.py +++ b/src/qlever/commands/benchmark_queries.py @@ -16,12 +16,12 @@ import yaml from termcolor import colored -from qlever import command_objects, engine_name, script_name +from qlever import command_objects from qlever.command import QleverCommand from qlever.commands.clear_cache import ClearCacheCommand -from qlever.commands.ui import dict_to_yaml from qlever.log import log, mute_log from qlever.util import ( + dict_to_yaml, pretty_printed_query, run_command, run_curl_command, @@ -279,7 +279,7 @@ def get_single_int_result(result_file: str) -> int | None: return single_int_result -def restart_server(start_only: bool = False) -> bool: +def restart_server(command_prefix: str, start_only: bool = False) -> bool: """ Restart the SPARQL server after the server hangs i.e. doesn't return results after timeout + 30s @@ -288,23 +288,23 @@ def restart_server(start_only: bool = False) -> bool: Only useful when Qleverfile in CWD and configured properly i.e. no command line args needed to call stop and start commands """ - stop_cmd = f"{script_name} stop" - start_cmd = f"{script_name} start" + stop_cmd = f"{command_prefix} stop" + start_cmd = f"{command_prefix} start" if not start_only: try: run_command(stop_cmd) time.sleep(2) except Exception as e: - log.warning(f"{script_name} process could not be stopped!: {e}") + log.warning(f"`{stop_cmd}` failed, server not stopped: {e}") try: run_command(start_cmd) time.sleep(5) - log.info(f"Successfully restarted {engine_name} server after hang!") + log.info("Successfully restarted the server after hang!") return True except Exception as e: log.warning( - f"{script_name} server could not be restarted. This might affect " - f"the benchmark process!: {e}" + f"`{start_cmd}` failed, server not restarted. This might " + f"affect the benchmark process: {e}" ) return False @@ -315,6 +315,7 @@ def resolve_benchmark_metadata( yml_name: str | None, yml_description: str | None, dataset: str | None, + command_prefix: str, ) -> tuple[str | None, str | None]: """ Resolve benchmark name and description using priority: @@ -324,7 +325,8 @@ def resolve_benchmark_metadata( """ dataset_name = dataset.capitalize() if dataset else None default_description = ( - f"{dataset_name} benchmark ran using {script_name} benchmark-queries" + f"{dataset_name} benchmark ran using " + f"{command_prefix} benchmark-queries" if dataset_name else None ) @@ -716,7 +718,7 @@ def additional_arguments(self, subparser) -> None: "the current engine, and resume execution with the next query. " "NOTE: This only works if all the server parameters for start and " "stop are configured in the Qleverfile and no arguments are needed " - f"for the {script_name} start and {script_name} stop commands." + "for the start and stop commands." ), ) @@ -879,6 +881,7 @@ def execute(self, args) -> bool: yml_name, yml_description, dataset, + args.command_prefix, ) # Launch the queries one after the other and for each print: the @@ -921,7 +924,7 @@ def execute(self, args) -> bool: with mute_log(): clear_cache_successful = ClearCacheCommand().execute(args) if not clear_cache_successful: - log.warn("Failed to clear the cache") + log.warning("Failed to clear the cache") # Remove OFFSET and LIMIT (after the last closing bracket). if args.remove_offset_and_limit or args.limit: @@ -1036,12 +1039,14 @@ def execute(self, args) -> bool: # If curl timed out after hitting max_time = 30s if "exit code 28" in str(e) and args.restart_on_hang: - server_restarted = restart_server() + server_restarted = restart_server(args.command_prefix) # If server is not responding and has crashed elif ( "exit code 52" in str(e) or "exit code 7" in str(e) ) and args.restart_on_hang: - server_restarted = restart_server(start_only=True) + server_restarted = restart_server( + args.command_prefix, start_only=True + ) if args.log_level == "DEBUG": traceback.print_exc() diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index 0aef33ee..fcd45e56 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -24,6 +24,8 @@ def render_usage_plot( settings_json: str, plot_max_points: int, plot_only: bool, + command_prefix: str, + engine_display: str, ) -> Path | None: """Render the resource-usage plot. @@ -43,7 +45,7 @@ def render_usage_plot( log.info( "To plot the resource-usage log, install matplotlib and " "numpy (`pip install qlever[plot]`), then run " - "`qlever index --resource-usage-plot-only`." + f"`{command_prefix} index --resource-usage-plot-only`." ) return None return usage_plot.render_usage_plot( @@ -51,6 +53,7 @@ def render_usage_plot( stxxl_memory=stxxl_memory, settings_json=settings_json, plot_max_points=plot_max_points, + engine_display=engine_display, ) @@ -239,6 +242,8 @@ def execute(self, args) -> bool: settings_json=args.settings_json, plot_max_points=args.resource_usage_plot_max_points, plot_only=True, + command_prefix=args.command_prefix, + engine_display=args.engine_display, ) if plot_path is None: return False @@ -278,7 +283,10 @@ def execute(self, args) -> bool: "multiple input streams)" ) log.info("") - log.info("See `qlever index --help` for more information") + log.info( + f"See `{args.command_prefix} index --help` for more " + "information" + ) return False # Add remaining options. @@ -356,7 +364,7 @@ def execute(self, args) -> bool: return False # Check if all of the input files exist. - if not input_files_exist(args.input_files): + if not input_files_exist(args.input_files, args.command_prefix): return False # Check if index files (name.index.*) already exist. @@ -413,6 +421,8 @@ def execute(self, args) -> bool: settings_json=args.settings_json, plot_max_points=args.resource_usage_plot_max_points, plot_only=False, + command_prefix=args.command_prefix, + engine_display=args.engine_display, ) if plot_path is not None: log.info(f"Resource-usage plot saved to `{plot_path.name}`") diff --git a/src/qlever/commands/index_stats.py b/src/qlever/commands/index_stats.py index 372a9a9a..28aa334c 100644 --- a/src/qlever/commands/index_stats.py +++ b/src/qlever/commands/index_stats.py @@ -78,19 +78,20 @@ def duration( # Compute durations for each indexing phase. Each entry maps a # phase name to (duration_in_time_unit, time_unit). - durations = {} - durations["Parse input"] = ( - duration([(overall_begin, merge_begin)]), - resolved_time_unit, - ) - durations["Build vocabularies"] = ( - duration([(merge_begin, convert_begin)]), - resolved_time_unit, - ) - durations["Convert to global IDs"] = ( - duration([(convert_begin, convert_end)]), - resolved_time_unit, - ) + durations = { + "Parse input": ( + duration([(overall_begin, merge_begin)]), + resolved_time_unit, + ), + "Build vocabularies": ( + duration([(merge_begin, convert_begin)]), + resolved_time_unit, + ), + "Convert to global IDs": ( + duration([(convert_begin, convert_end)]), + resolved_time_unit, + ), + } for name, perm_begin, perm_end in iter_permutation_phases( permutations, normal_end ): @@ -151,9 +152,10 @@ def compute_sizes( unit_factor = get_size_unit_factor(size_unit) sizes = {k: v / unit_factor for k, v in raw_sizes.items()} - sizes_to_show = {} - sizes_to_show["Files index.*"] = (sizes["index"], size_unit) - sizes_to_show["Files vocabulary.*"] = (sizes["vocabulary"], size_unit) + sizes_to_show = { + "Files index.*": (sizes["index"], size_unit), + "Files vocabulary.*": (sizes["vocabulary"], size_unit), + } if sizes["text"] > 0: sizes_to_show["Files text.*"] = (sizes["text"], size_unit) sizes_to_show["TOTAL size"] = (sizes["total"], size_unit) diff --git a/src/qlever/commands/settings.py b/src/qlever/commands/settings.py index 3c19358d..adbe2f7f 100644 --- a/src/qlever/commands/settings.py +++ b/src/qlever/commands/settings.py @@ -19,7 +19,7 @@ def __init__(self): pass def description(self) -> str: - return "Show or set server settings (after `qlever start`)" + return "Show or set server settings (after the server has started)" def should_have_qleverfile(self) -> bool: return True diff --git a/src/qlever/commands/setup_config.py b/src/qlever/commands/setup_config.py index a558a47f..fc33f9eb 100644 --- a/src/qlever/commands/setup_config.py +++ b/src/qlever/commands/setup_config.py @@ -50,15 +50,15 @@ def additional_arguments(self, subparser) -> None: help="The name of the pre-configured Qleverfile to create", ) - def check_qleverfile_exists(self) -> bool: + def check_qleverfile_exists(self, command_prefix: str) -> bool: """Return True if a Qleverfile already exists (and log an error).""" if Path("Qleverfile").exists(): log.error("`Qleverfile` already exists in current directory") log.info("") log.info( "If you want to create a new Qleverfile using " - "`qlever setup-config`, delete the existing Qleverfile " - "first" + f"`{command_prefix} setup-config`, delete the existing " + "Qleverfile first" ) return True return False @@ -94,7 +94,7 @@ def execute(self, args) -> bool: if args.show: return True - if self.check_qleverfile_exists(): + if self.check_qleverfile_exists(args.command_prefix): return False # Copy the Qleverfile to the current directory. diff --git a/src/qlever/commands/start.py b/src/qlever/commands/start.py index 6f858270..9a480d38 100644 --- a/src/qlever/commands/start.py +++ b/src/qlever/commands/start.py @@ -154,7 +154,7 @@ def __init__(self): def description(self) -> str: return ( "Start the QLever server (requires that you have built " - "an index with `qlever index` before)" + "an index with the `index` command before)" ) def should_have_qleverfile(self) -> bool: @@ -276,8 +276,8 @@ def execute(self, args) -> bool: log.error(f"QLever server already running on {args.endpoint_url}") log.info("") log.info( - "To kill the existing server, use `qlever stop` " - "or `qlever start` with option " + f"To kill the existing server, use `{args.command_prefix} " + f"stop` or `{args.command_prefix} start` with option " "--kill-existing-with-same-port`" ) @@ -398,7 +398,7 @@ def execute(self, args) -> bool: try: process.wait() except KeyboardInterrupt: - log.warn("\rCtrl-C pressed, stopping the server ...") + log.warning("\rCtrl-C pressed, stopping the server ...") log.info("") process.terminate() # Stop the container process manually diff --git a/src/qlever/commands/system_info.py b/src/qlever/commands/system_info.py index e8b71a2b..0ab38c5a 100644 --- a/src/qlever/commands/system_info.py +++ b/src/qlever/commands/system_info.py @@ -69,8 +69,11 @@ def execute(self, args) -> bool: is_mac = system == "Darwin" is_windows = system == "Windows" if is_windows: - log.warn("Only limited information is gathered on Windows.") - log.info(f"Version: {version('qlever')} (qlever --version)") + log.warning("Only limited information is gathered on Windows.") + # `--version` sits on the top-level parser only, so the command to + # show it is the first word of the prefix (`qlever`, `qeval`). + script_name = args.command_prefix.split()[0] + log.info(f"Version: {version('qlever')} ({script_name} --version)") if is_linux: info = platform.freedesktop_os_release() log.info(f"OS: {platform.system()} ({info['PRETTY_NAME']})") diff --git a/src/qlever/commands/ui.py b/src/qlever/commands/ui.py index c0b821b3..7b2868d4 100644 --- a/src/qlever/commands/ui.py +++ b/src/qlever/commands/ui.py @@ -8,34 +8,7 @@ from qlever.command import QleverCommand from qlever.containerize import Containerize from qlever.log import log -from qlever.util import is_port_used, run_command - - -# Return a YAML string for the given dictionary. Format values with -# newlines using the "|" style. -def dict_to_yaml(dictionary: dict) -> str: - """ - Custom representer for yaml, which uses the "|" style only for - multiline strings. - - NOTE: We replace all `\r\n` with `\n` because otherwise the `|` style - does not work as expected. - """ - - class MultiLineDumper(yaml.SafeDumper): - def represent_scalar(self, tag, value, style=None): - value = value.replace("\r\n", "\n") - if isinstance(value, str) and "\n" in value: - style = "|" - return super().represent_scalar(tag, value, style) - - # Dump as yaml. - return yaml.dump( - dictionary, - sort_keys=False, - allow_unicode=True, - Dumper=MultiLineDumper, - ) +from qlever.util import dict_to_yaml, is_port_used, run_command class UiCommand(QleverCommand): @@ -98,15 +71,15 @@ def execute(self, args) -> bool: ) if qlever_is_running_in_container: log.error( - "The environment variable `QLEVER_OVERRIDE_DISABLE_UI` is set, " - "therefore `qlever ui` is not available (it should not be called " - "from inside a container)" + "The environment variable `QLEVER_OVERRIDE_DISABLE_UI` is " + f"set, therefore `{args.command_prefix} ui` is not available " + "(it should not be called from inside a container)" ) log.info("") if not args.show: log.info( "For your information, showing the commands that are " - "executed when `qlever ui` is available:" + f"executed when `{args.command_prefix} ui` is available:" ) log.info("") @@ -265,6 +238,6 @@ def execute(self, args) -> bool: ) log.info( f"You can modify the config file at `{ui_config_file}` " - f"and then just run `qlever ui` again" + f"and then just run `{args.command_prefix} ui` again" ) return True diff --git a/src/qlever/commands/update_wikidata.py b/src/qlever/commands/update_wikidata.py index db03f1ca..1c47377d 100644 --- a/src/qlever/commands/update_wikidata.py +++ b/src/qlever/commands/update_wikidata.py @@ -324,7 +324,7 @@ def retry_with_backoff(self, operation, operation_name, max_retries): delay_str = f"{retry_delay // 60}min" else: delay_str = f"{retry_delay}s" - log.warn( + log.warning( f"{operation_name} failed (attempt {attempt + 1}/{max_retries}): {e}. " f"Retrying in {delay_str} ..." ) @@ -353,19 +353,21 @@ def iter_sse_events(source): try: yield from source except Exception as e: - log.warn(f"SSE stream connection lost ({e}), will reconnect ...") + log.warning( + f"SSE stream connection lost ({e}), will reconnect ..." + ) def determine_batch_size_for_cached_update( self, offset: int, batch_size: int ) -> int | None: options = list(Path.cwd().glob(f"update.{offset}.*.sparql")) if len(options) == 0: - log.warn( + log.warning( "Found no cached SPARQL update. Continuing with update stream." ) return None elif len(options) > 1: - log.warn( + log.warning( f"Found {len(options)} candidates for cached SPARQL update. Using {options[0].name}." ) return int( @@ -539,7 +541,7 @@ def execute(self, args) -> bool: # Special handling of Ctrl+C, see `handle_ctrl_c` above. signal.signal(signal.SIGINT, self.handle_ctrl_c) - log.warn("Press Ctrl+C to finish and exit gracefully") + log.warning("Press Ctrl+C to finish and exit gracefully") log.info("") # If no `--offset` is provided, try to get the offset from @@ -584,7 +586,7 @@ def execute(self, args) -> bool: ) args.offset = offset except KeyboardInterrupt: - log.warn( + log.warning( "\rCtrl+C pressed while determine current state, exiting" ) return True @@ -648,7 +650,7 @@ def execute(self, args) -> bool: wait_before_next_batch = False self.ctrl_c_pressed.wait(args.wait_between_batches) if self.ctrl_c_pressed.is_set(): - log.warn( + log.warning( "\rCtrl+C pressed while waiting in between batches, " "exiting" ) @@ -686,7 +688,7 @@ def execute(self, args) -> bool: args.num_retries, ) except KeyboardInterrupt: - log.warn( + log.warning( "\rCtrl+C pressed while while connecting to stream, " "exiting" ) @@ -732,7 +734,7 @@ def execute(self, args) -> bool: args.num_retries, ) except KeyboardInterrupt: - log.warn( + log.warning( "\rCtrl+C pressed while while verifying state, exiting" ) break @@ -907,7 +909,7 @@ def execute(self, args) -> bool: and entity_id in delete_entity_ids ): if args.verbose == "yes": - log.warn( + log.warning( f"Encountered operation that adds data for " f"an entity ID ({entity_id}) that was deleted " f"earlier in this batch; finishing batch and " @@ -936,7 +938,7 @@ def execute(self, args) -> bool: and current_batch_size > 0 ): if args.verbose == "yes": - log.warn( + log.warning( f"Encountered message with date {date}, which is within " f"{args.lag_seconds} " f"second{'s' if args.lag_seconds > 1 else ''} " @@ -955,7 +957,7 @@ def execute(self, args) -> bool: and date >= args.until and current_batch_size > 0 ): - log.warn( + log.warning( f"Reached --until date {args.until} " f"(message date: {date}), that's it folks" ) @@ -1169,7 +1171,7 @@ def node_to_sparql(node: rdflib.term.Node) -> str: # end of the inner event loop so that always at least one # message is processed). if self.ctrl_c_pressed.is_set(): - log.warn( + log.warning( "\rCtrl+C pressed while processing a batch, " "finishing it and exiting" ) @@ -1311,12 +1313,12 @@ def node_to_sparql(node: rdflib.term.Node) -> str: result = run_command(curl_cmd, return_output=True) except Exception: if self.ctrl_c_pressed.is_set(): - log.warn( + log.warning( "\r \nCtrl+C pressed while executing update, exiting" ) return True else: - log.warn( + log.warning( "\r \nUpdate request failed; will reconnect and retry" ) event_id_for_next_batch = [ @@ -1526,7 +1528,7 @@ def get_time_ms( ) except Exception as e: - log.warn( + log.warning( f"Error extracting statistics: {e}, " f"curl command was: {curl_cmd}" ) diff --git a/src/qlever/config.py b/src/qlever/config.py index cbc6235d..cf09b6b0 100644 --- a/src/qlever/config.py +++ b/src/qlever/config.py @@ -145,7 +145,7 @@ def parse_args(self): argcomplete_enabled = os.environ.get("QLEVER_ARGCOMPLETE_ENABLED") if not argcomplete_enabled and not argcomplete_check_off: log.info("") - log.warn( + log.warning( f"To enable autocompletion, run the following command, " f"and consider adding it to your `.bashrc` or `.zshrc`:" f"\n\n" @@ -197,7 +197,7 @@ def add_qleverfile_option(parser): # we then parse the Qleverfile or not. if qleverfile_exists and not autocomplete_mode: try: - qleverfile_config = Qleverfile.read(qleverfile_path) + qleverfile_config = Qleverfile.read(qleverfile_path, "qlever") except Exception as e: log.info("") log.error(f"Error parsing Qleverfile `{qleverfile_path}`: {e}") @@ -217,6 +217,14 @@ def add_qleverfile_option(parser): attrs=["bold"], ) ) + # `engine` keys machine-facing things like container names, + # `command_prefix` is what the user types before a command, and + # `engine_display` is the human-readable name. + parser.set_defaults( + engine="qlever", + engine_display="QLever", + command_prefix="qlever", + ) if script_name == "qlever": parser.add_argument( "--version", @@ -226,7 +234,7 @@ def add_qleverfile_option(parser): add_qleverfile_option(parser) subparsers = parser.add_subparsers(dest="command") subparsers.required = True - all_args = Qleverfile.all_arguments() + all_args = Qleverfile.all_arguments("qlever") for command_name, command_object in command_objects.items(): self.add_subparser_for_command( subparsers, diff --git a/src/qlever/qlever_main.py b/src/qlever/qlever_main.py index 50284916..d03ec8f5 100644 --- a/src/qlever/qlever_main.py +++ b/src/qlever/qlever_main.py @@ -48,7 +48,7 @@ def main(): if not command_successful: exit(1) except KeyboardInterrupt: - log.warn("\rCtrl-C pressed, exiting ...") + log.warning("\rCtrl-C pressed, exiting ...") log.info("") exit(1) except Exception as e: diff --git a/src/qlever/qleverfile.py b/src/qlever/qleverfile.py index c254e77c..a56df08e 100644 --- a/src/qlever/qleverfile.py +++ b/src/qlever/qleverfile.py @@ -5,10 +5,8 @@ import subprocess from argparse import ArgumentTypeError from configparser import ConfigParser, ExtendedInterpolation, RawConfigParser -from importlib import import_module from pathlib import Path -from qlever import script_name from qlever.containerize import Containerize from qlever.log import log from qlever.util import positive_int @@ -85,7 +83,7 @@ class Qleverfile: ] @staticmethod - def all_arguments(): + def all_arguments(command_prefix: str) -> dict: """ Define all possible parameters. A value of `None` means that there is no default value. @@ -379,7 +377,8 @@ def arg(*args, **kwargs): default="manual", help="When to rebuild the index from the current data (including " 'updates): "manual" (only when explicitly requested via ' - '`qlever rebuild-index`) or "automatic:min:max:fraction" (additionally ' + f"`{command_prefix} rebuild-index`) or " + '"automatic:min:max:fraction" (additionally ' "rebuild automatically in the background once the number of delta " "triples reaches the given `fraction` of the number of index " "triples, but never below `min` and always at `max`, " @@ -416,7 +415,7 @@ def arg(*args, **kwargs): choices=["yes", "no"], default="yes", help="Whether to use the patterns precomputed during the index " - "build (see `qlever index --help` for their utility)", + f"build (see `{command_prefix} index --help` for their utility)", ) server_args["metrics_log"] = arg( "--metrics-log", @@ -443,7 +442,7 @@ def arg(*args, **kwargs): choices=["yes", "no"], default="no", help="Whether to use the text index (requires that one was " - "built, see `qlever index`)", + f"built, see `{command_prefix} index`)", ) server_args["preload_materialized_views"] = arg( "-l", @@ -457,8 +456,8 @@ def arg(*args, **kwargs): "--warmup-cmd", type=str, help="Command executed after the server has started " - " (executed as part of `qlever start` unless " - " `--no-warmup` is specified, or with `qlever warmup`)", + f" (executed as part of `{command_prefix} start` unless " + f" `--no-warmup` is specified, or with `{command_prefix} warmup`)", ) server_args["enable_metrics"] = arg( "--enable-metrics", @@ -487,12 +486,12 @@ def arg(*args, **kwargs): runtime_args["index_container"] = arg( "--index-container", type=str, - help=f"The name of the container used by `{script_name} index`", + help=f"The name of the container used by `{command_prefix} index`", ) runtime_args["server_container"] = arg( "--server-container", type=str, - help=f"The name of the container used by `{script_name} start`", + help=f"The name of the container used by `{command_prefix} start`", ) runtime_args["restart_policy"] = arg( "--restart-policy", @@ -508,7 +507,9 @@ def arg(*args, **kwargs): "--ui-port", type=int, default=8176, - help="The port of the Qlever UI when running `qlever ui`", + help=( + f"The port of the Qlever UI when running `{command_prefix} ui`" + ), ) ui_args["ui_config"] = arg( "--ui-config", @@ -522,36 +523,29 @@ def arg(*args, **kwargs): type=str, choices=Containerize.supported_systems(), default="docker", - help="Which container system to use for `qlever ui`" - " (unlike for `qlever index` and `qlever start`, " - ' "native" is not yet supported here)', + help=( + f"Which container system to use for `{command_prefix} ui` " + f"(unlike for `{command_prefix} index` and " + f'`{command_prefix} start`, "native" is not yet supported ' + "here)" + ), ) ui_args["ui_image"] = arg( "--ui-image", type=str, default="docker.io/adfreiburg/qlever-ui", - help="The name of the image used for `qlever ui`", + help=f"The name of the image used for `{command_prefix} ui`", ) ui_args["ui_container"] = arg( "--ui-container", type=str, - help="The name of the container used for `qlever ui`", + help=f"The name of the container used for `{command_prefix} ui`", ) - engine_args_module_path = f"{script_name}.qleverfile" - try: - if script_name != "qlever": - module = import_module(engine_args_module_path) - module.qleverfile_args(all_args) - except (ImportError, AttributeError) as e: - log.debug( - f"Could not import module {engine_args_module_path}: {e}" - ) - return all_args @staticmethod - def read(qleverfile_path): + def read(qleverfile_path: Path, engine: str) -> ConfigParser: """ Read the given Qleverfile (the function assumes that it exists) and return a `ConfigParser` object with all the options and their values. @@ -604,21 +598,22 @@ def read(qleverfile_path): config[section] = {} # Add default values that are based on other values. + index = config["index"] + server = config["server"] if "name" in config["data"]: name = config["data"]["name"] runtime = config["runtime"] if "server_container" not in runtime: - runtime["server_container"] = f"{script_name}.server.{name}" + runtime["server_container"] = f"{engine}.server.{name}" if "index_container" not in runtime: - runtime["index_container"] = f"{script_name}.index.{name}" + runtime["index_container"] = f"{engine}.index.{name}" if "ui_container" not in config["ui"]: config["ui"]["ui_container"] = f"qlever.ui.{name}" - index = config["index"] if "text_words_file" not in index: index["text_words_file"] = f"{name}.wordsfile.tsv" if "text_docs_file" not in index: index["text_docs_file"] = f"{name}.docsfile.tsv" - server = config["server"] + if index.get("text_index", "none") != "none": server["use_text_index"] = "yes" if index.get("only_pso_and_pos_permutations", "false") == "true": diff --git a/src/qlever/resource_usage/usage_plot.py b/src/qlever/resource_usage/usage_plot.py index 9a447efc..cc1307ea 100644 --- a/src/qlever/resource_usage/usage_plot.py +++ b/src/qlever/resource_usage/usage_plot.py @@ -13,7 +13,6 @@ matplotlib.use("Agg") from matplotlib import pyplot as plt # noqa: E402 -from qlever import engine_name from qlever.log import log from qlever.util import ( iter_permutation_phases, @@ -308,6 +307,7 @@ def write_usage_plot( def render_usage_plot( dataset: str, + engine_display: str, stxxl_memory: str = "", settings_json: str = "{}", output_dir: Path | None = None, @@ -337,7 +337,7 @@ def render_usage_plot( stxxl_memory=stxxl_memory, settings_json=settings_json, out_path=plot_path, - title=f"{engine_name} index build: {dataset}", + title=f"{engine_display} index build: {dataset}", plot_max_points=plot_max_points, ) except Exception as error: diff --git a/src/qlever/util.py b/src/qlever/util.py index 0dfb2658..563030c5 100644 --- a/src/qlever/util.py +++ b/src/qlever/util.py @@ -17,11 +17,32 @@ from typing import Any, NamedTuple, Optional import psutil +import yaml -from qlever import script_name from qlever.log import log +def dict_to_yaml(dictionary: dict) -> str: + """ + Dump a dict to YAML, using the `|` block style for multiline strings. + """ + + class MultiLineDumper(yaml.SafeDumper): + def represent_scalar(self, tag, value, style=None): + # The `|` style does not work as expected with `\r\n`. + value = value.replace("\r\n", "\n") + if isinstance(value, str) and "\n" in value: + style = "|" + return super().represent_scalar(tag, value, style) + + return yaml.dump( + dictionary, + sort_keys=False, + allow_unicode=True, + Dumper=MultiLineDumper, + ) + + def get_total_file_size( patterns: list[str], exclude: set[str] | None = None ) -> int: @@ -404,7 +425,7 @@ def binary_exists(binary: str, cmd_arg: str, args) -> bool: is_containerized = args.system in Containerize.supported_systems() cmd = f"{binary} --help" - if is_containerized and script_name == "qlever": + if is_containerized and args.engine == "qlever": cmd = Containerize().containerize_command( cmd, args.system, @@ -452,7 +473,7 @@ def is_server_alive(url: str) -> bool: return False -def input_files_exist(input_files: str) -> bool: +def input_files_exist(input_files: str, command_prefix: str) -> bool: """ Check if all of the input files exist in current working directory. """ @@ -461,7 +482,7 @@ def input_files_exist(input_files: str) -> bool: log.error(f'No file matching "{pattern}" found') log.info("") log.info( - f"Did you call `{script_name} get-data`? If you did, " + f"Did you call `{command_prefix} get-data`? If you did, " "check GET_DATA_CMD and INPUT_FILES in the Qleverfile" ) return False diff --git a/test/qlever/commands/test_benchmark_queries_methods.py b/test/qlever/commands/test_benchmark_queries_methods.py index 1cf1db3f..ea809b06 100644 --- a/test/qlever/commands/test_benchmark_queries_methods.py +++ b/test/qlever/commands/test_benchmark_queries_methods.py @@ -463,7 +463,7 @@ def test_parse_queries_tsv_command_failure(mock_command): ) def test_resolve_benchmark_metadata(case): name, desc = resolve_benchmark_metadata( - *case["cli"], *case["yml"], case["dataset"] + *case["cli"], *case["yml"], case["dataset"], "qlever" ) exp_name, exp_desc = case["expected"] assert name == exp_name diff --git a/test/qlever/commands/test_index_execute.py b/test/qlever/commands/test_index_execute.py index fa8a45f5..6281b96f 100644 --- a/test/qlever/commands/test_index_execute.py +++ b/test/qlever/commands/test_index_execute.py @@ -335,6 +335,7 @@ def test_execute_cat_files_and_multi_json(self, mock_log): args.resource_usage_plot_only = False args.cat_input_files = True args.multi_input_json = True + args.command_prefix = "qlever" # Instantiate IndexCommand and execute the function result = IndexCommand().execute(args) diff --git a/test/qlever/commands/test_start_other_methods.py b/test/qlever/commands/test_start_other_methods.py index 437b249f..73d7800a 100644 --- a/test/qlever/commands/test_start_other_methods.py +++ b/test/qlever/commands/test_start_other_methods.py @@ -11,7 +11,7 @@ def test_description(self): StartCommand().description(), "Start the " "QLever server (requires that you have built " - "an index with `qlever index` before)", + "an index with the `index` command before)", ) def test_should_have_qleverfile(self): @@ -94,7 +94,7 @@ def test_additional_arguments(self): self.assertEqual(argument_help, "Do not execute the warmup command") def test_preload_materialized_views_qleverfile_argument(self): - args, kwargs = Qleverfile.all_arguments()["server"][ + args, kwargs = Qleverfile.all_arguments("qlever")["server"][ "preload_materialized_views" ] diff --git a/test/qlever/resource_usage/test_usage_plot.py b/test/qlever/resource_usage/test_usage_plot.py index c0efa55e..b581e897 100644 --- a/test/qlever/resource_usage/test_usage_plot.py +++ b/test/qlever/resource_usage/test_usage_plot.py @@ -210,7 +210,7 @@ def test_compute_phase_boundaries_skips_incomplete_phase(tmp_path): def test_render_usage_plot_missing_tsv(tmp_path): - assert render_usage_plot("missing", output_dir=tmp_path) is None + assert render_usage_plot("missing", "QLever", output_dir=tmp_path) is None def test_render_usage_plot_header_only_tsv_renders_nothing(tmp_path): @@ -218,7 +218,7 @@ def test_render_usage_plot_header_only_tsv_renders_nothing(tmp_path): # or leave a PNG behind. tsv_path = tmp_path / "data.index.resource-usage-log.tsv" tsv_path.write_text("elapsed_s\trss\tcpu_percent\n") - assert render_usage_plot("data", output_dir=tmp_path) is None + assert render_usage_plot("data", "QLever", output_dir=tmp_path) is None assert not (tmp_path / "data.resource-usage-plot.png").exists() @@ -228,6 +228,6 @@ def test_render_usage_plot_falls_back_to_old_tsv_name(tmp_path): tsv_path.write_text( "elapsed_s\trss\tcpu_percent\n1.0\t100\t5.0\n2.0\t200\t6.0\n" ) - plot_path = render_usage_plot("data", output_dir=tmp_path) + plot_path = render_usage_plot("data", "QLever", output_dir=tmp_path) assert plot_path == tmp_path / "data.resource-usage-plot.png" assert plot_path.exists() From 3f7a67df87d4a0edd4cf0ae022cba2936d8865ac Mon Sep 17 00:00:00 2001 From: tanmay-9 Date: Sat, 15 Aug 2026 16:00:15 +0200 Subject: [PATCH 2/7] Apply ruff 0.16 automatic fixes --- src/qlever/commands/benchmark_queries.py | 4 +--- src/qlever/commands/cache_stats.py | 4 ++-- src/qlever/commands/materialized_view.py | 1 - src/qlever/commands/serve_evaluation_app.py | 2 +- src/qlever/commands/setup_config.py | 2 +- src/qlever/containerize.py | 5 ++--- src/qlever/qleverfile.py | 1 - src/qlever/resource_usage/usage_plot.py | 2 +- src/qlever/util.py | 4 ++-- test/qlever/test_footer.py | 8 ++++---- 10 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/qlever/commands/benchmark_queries.py b/src/qlever/commands/benchmark_queries.py index 5ac44b4f..b1952e83 100644 --- a/src/qlever/commands/benchmark_queries.py +++ b/src/qlever/commands/benchmark_queries.py @@ -453,9 +453,7 @@ def get_result_yml_query_record( headers = [] if result_size is not None and isinstance(result, str): record["result_size"] = result_size - result_size = ( - max_result_size if result_size > max_result_size else result_size - ) + result_size = min(result_size, max_result_size) headers, results = get_query_results( result, result_size, accept_header ) diff --git a/src/qlever/commands/cache_stats.py b/src/qlever/commands/cache_stats.py index c29e9836..451de3c0 100644 --- a/src/qlever/commands/cache_stats.py +++ b/src/qlever/commands/cache_stats.py @@ -122,9 +122,9 @@ def show_dict_as_table(key_value_pairs): max_key_len = max([len(key) for key, _ in key_value_pairs]) for key, value in key_value_pairs: if isinstance(value, int) or re.match(r"^\d+$", value): - value = "{:,}".format(int(value)) + value = f"{int(value):,}" if re.match(r"^\d+\.\d+$", value): - value = "{:.2f}".format(float(value)) + value = f"{float(value):.2f}" log.info(f"{key.ljust(max_key_len)} : {value}") show_dict_as_table(cache_stats_dict.items()) diff --git a/src/qlever/commands/materialized_view.py b/src/qlever/commands/materialized_view.py index 531dc470..858e33d9 100644 --- a/src/qlever/commands/materialized_view.py +++ b/src/qlever/commands/materialized_view.py @@ -19,7 +19,6 @@ class MaterializedViewCommand(QleverCommand): def __init__(self): self.materialized_view_name_regex = r"^[A-Za-z0-9-]+$" - pass def description(self) -> str: return ( diff --git a/src/qlever/commands/serve_evaluation_app.py b/src/qlever/commands/serve_evaluation_app.py index 76e93d7f..1127a9f5 100644 --- a/src/qlever/commands/serve_evaluation_app.py +++ b/src/qlever/commands/serve_evaluation_app.py @@ -164,7 +164,7 @@ def do_GET(self) -> None: except Exception as e: self.send_response(500) self.end_headers() - self.wfile.write(f"Error loading YAMLs: {e}".encode("utf-8")) + self.wfile.write(f"Error loading YAMLs: {e}".encode()) else: super().do_GET() diff --git a/src/qlever/commands/setup_config.py b/src/qlever/commands/setup_config.py index fc33f9eb..504614b3 100644 --- a/src/qlever/commands/setup_config.py +++ b/src/qlever/commands/setup_config.py @@ -4,7 +4,7 @@ from os import environ from pathlib import Path -import qlever.util as util +from qlever import util from qlever.command import QleverCommand from qlever.log import log diff --git a/src/qlever/containerize.py b/src/qlever/containerize.py index 11150ba7..3e1dd571 100644 --- a/src/qlever/containerize.py +++ b/src/qlever/containerize.py @@ -6,7 +6,6 @@ import shlex import subprocess -from typing import Optional from qlever.log import log from qlever.util import get_random_string, run_command @@ -39,7 +38,7 @@ def containerize_command( container_name: str, volumes: list[tuple[str, str]] = [], ports: list[tuple[int, int]] = [], - working_directory: Optional[str] = None, + working_directory: str | None = None, use_bash: bool = True, ) -> str: """ @@ -148,7 +147,7 @@ def stop_and_remove_container( return False @staticmethod - def run_in_container(cmd: str, args) -> Optional[str]: + def run_in_container(cmd: str, args) -> str | None: """ Run an arbitrary command in the qlever container and return its output. """ diff --git a/src/qlever/qleverfile.py b/src/qlever/qleverfile.py index a56df08e..51f4eef1 100644 --- a/src/qlever/qleverfile.py +++ b/src/qlever/qleverfile.py @@ -629,7 +629,6 @@ def read(qleverfile_path: Path, engine: str) -> ConfigParser: log.warning( "Could not get the hostname, using `localhost` as default" ) - pass # Return the parsed Qleverfile with the added inherited values. return config diff --git a/src/qlever/resource_usage/usage_plot.py b/src/qlever/resource_usage/usage_plot.py index cc1307ea..a0510c7a 100644 --- a/src/qlever/resource_usage/usage_plot.py +++ b/src/qlever/resource_usage/usage_plot.py @@ -11,7 +11,7 @@ import psutil matplotlib.use("Agg") -from matplotlib import pyplot as plt # noqa: E402 +from matplotlib import pyplot as plt from qlever.log import log from qlever.util import ( diff --git a/src/qlever/util.py b/src/qlever/util.py index 563030c5..d14c2fc2 100644 --- a/src/qlever/util.py +++ b/src/qlever/util.py @@ -14,7 +14,7 @@ from collections.abc import Iterator from datetime import date, datetime from pathlib import Path -from typing import Any, NamedTuple, Optional +from typing import Any, NamedTuple import psutil import yaml @@ -68,7 +68,7 @@ def run_command( show_output: bool = False, show_stderr: bool = False, use_popen: bool = False, -) -> Optional[str | subprocess.Popen]: +) -> str | subprocess.Popen | None: """ Run the given command and throw an exception if the exit code is non-zero. If `return_output` is `True`, return what the command wrote to `stdout`. diff --git a/test/qlever/test_footer.py b/test/qlever/test_footer.py index ad64d3d4..7f17e267 100644 --- a/test/qlever/test_footer.py +++ b/test/qlever/test_footer.py @@ -14,11 +14,11 @@ pytest.importorskip("textual") -from textual import events # noqa: E402 -from textual.app import App # noqa: E402 -from textual.binding import Binding # noqa: E402 +from textual import events +from textual.app import App +from textual.binding import Binding -from qlever.monitor_queries.widgets.footer import Footer # noqa: E402 +from qlever.monitor_queries.widgets.footer import Footer class CountingFooter(Footer): From 86e35551a2af0acd8b5e04a9405ca6ec54ee1393 Mon Sep 17 00:00:00 2001 From: tanmay-9 Date: Sat, 15 Aug 2026 16:14:08 +0200 Subject: [PATCH 3/7] Remove redundant pytest import and importorskip call from test_footer.py --- test/qlever/test_footer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/qlever/test_footer.py b/test/qlever/test_footer.py index 7f17e267..ca312f52 100644 --- a/test/qlever/test_footer.py +++ b/test/qlever/test_footer.py @@ -10,10 +10,6 @@ import asyncio -import pytest - -pytest.importorskip("textual") - from textual import events from textual.app import App from textual.binding import Binding From 50f469a48394a86c8193ca94e830b8775ed06ade Mon Sep 17 00:00:00 2001 From: tanmay-9 Date: Sat, 15 Aug 2026 16:29:33 +0200 Subject: [PATCH 4/7] Remove not-yet accurate comment --- src/qlever/commands/system_info.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/qlever/commands/system_info.py b/src/qlever/commands/system_info.py index 0ab38c5a..fcc9422d 100644 --- a/src/qlever/commands/system_info.py +++ b/src/qlever/commands/system_info.py @@ -70,8 +70,6 @@ def execute(self, args) -> bool: is_windows = system == "Windows" if is_windows: log.warning("Only limited information is gathered on Windows.") - # `--version` sits on the top-level parser only, so the command to - # show it is the first word of the prefix (`qlever`, `qeval`). script_name = args.command_prefix.split()[0] log.info(f"Version: {version('qlever')} ({script_name} --version)") if is_linux: From c290769ae1994c113fa4d98b8132de4f3868d061 Mon Sep 17 00:00:00 2001 From: tanmay-9 Date: Mon, 17 Aug 2026 12:22:31 +0200 Subject: [PATCH 5/7] Add ResourceMonitor for the new engines and improve usage_plot so that new engines can reuse existing code --- src/qlever/commands/index.py | 42 +-- src/qlever/resource_usage/resource_monitor.py | 260 ++++++++++++++++++ src/qlever/resource_usage/usage_plot.py | 143 +++++++--- .../resource_usage/test_resource_monitor.py | 59 ++++ test/qlever/resource_usage/test_usage_plot.py | 91 +++++- 5 files changed, 526 insertions(+), 69 deletions(-) create mode 100644 src/qlever/resource_usage/resource_monitor.py create mode 100644 test/qlever/resource_usage/test_resource_monitor.py diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index fcd45e56..695c776a 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -18,15 +18,7 @@ ) -def render_usage_plot( - dataset: str, - stxxl_memory: str, - settings_json: str, - plot_max_points: int, - plot_only: bool, - command_prefix: str, - engine_display: str, -) -> Path | None: +def render_usage_plot(args, plot_only: bool) -> Path | None: """Render the resource-usage plot. When the plotting libraries are missing, this is an error if the @@ -45,15 +37,13 @@ def render_usage_plot( log.info( "To plot the resource-usage log, install matplotlib and " "numpy (`pip install qlever[plot]`), then run " - f"`{command_prefix} index --resource-usage-plot-only`." + f"`{args.command_prefix} index --resource-usage-plot-only`." ) return None return usage_plot.render_usage_plot( - dataset, - stxxl_memory=stxxl_memory, - settings_json=settings_json, - plot_max_points=plot_max_points, - engine_display=engine_display, + args, + overlay=usage_plot.qlever_overlay, + subtitle=usage_plot.qlever_subtitle, ) @@ -236,15 +226,7 @@ def execute(self, args) -> bool: # Render the resource-usage plot from the existing log without # rebuilding the index. if args.resource_usage_plot_only: - plot_path = render_usage_plot( - args.name, - stxxl_memory=args.stxxl_memory or "", - settings_json=args.settings_json, - plot_max_points=args.resource_usage_plot_max_points, - plot_only=True, - command_prefix=args.command_prefix, - engine_display=args.engine_display, - ) + plot_path = render_usage_plot(args, plot_only=True) if plot_path is None: return False log.info(f"Resource-usage plot saved to `{plot_path.name}`") @@ -363,7 +345,7 @@ def execute(self, args) -> bool: if not binary_exists(args.index_binary, "index-binary", args): return False - # Check if all of the input files exist. + # Check if all the input files exist. if not input_files_exist(args.input_files, args.command_prefix): return False @@ -415,15 +397,7 @@ def execute(self, args) -> bool: Path(f"{args.name}.index.resource-usage-log.tsv").exists() or Path(f"{args.name}.resource-usage-log.tsv").exists() ): - plot_path = render_usage_plot( - args.name, - stxxl_memory=args.stxxl_memory or "", - settings_json=args.settings_json, - plot_max_points=args.resource_usage_plot_max_points, - plot_only=False, - command_prefix=args.command_prefix, - engine_display=args.engine_display, - ) + plot_path = render_usage_plot(args, plot_only=False) if plot_path is not None: log.info(f"Resource-usage plot saved to `{plot_path.name}`") diff --git a/src/qlever/resource_usage/resource_monitor.py b/src/qlever/resource_usage/resource_monitor.py new file mode 100644 index 00000000..5735cfbc --- /dev/null +++ b/src/qlever/resource_usage/resource_monitor.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, fields +from pathlib import Path + +import psutil + +from qlever.containerize import Containerize +from qlever.log import log +from qlever.util import ( + container_memory_to_bytes, + find_process_by_binary, + run_command, +) + + +@dataclass +class Sample: + """One sample of elapsed time, memory (RSS), and CPU usage; None + fields are written as empty TSV columns.""" + + elapsed_s: float | None = None + rss: int | None = None + cpu_percent: float | None = None + + +def sample_to_tsv_row(sample: Sample) -> str: + """Format a Sample as a TSV row; None fields become empty columns.""" + values = [getattr(sample, field.name) for field in fields(sample)] + return "\t".join("" if v is None else str(v) for v in values) + "\n" + + +def sample_process(proc: psutil.Process) -> Sample: + """ + One RSS+CPU read from a psutil.Process; empty Sample on access errors. + """ + try: + mem = proc.memory_info() + cpu_pct = proc.cpu_percent(interval=None) + except (psutil.NoSuchProcess, psutil.AccessDenied): + return Sample() + return Sample(rss=mem.rss, cpu_percent=cpu_pct) + + +def sample_container(system: str, container: str) -> Sample: + """ + One RSS+CPU read via ` stats --no-stream` on a named container. + """ + try: + output = run_command( + f"{system} stats --no-stream" + f" --format '{{{{.MemUsage}}}}\t{{{{.CPUPerc}}}}'" + f" {container}", + return_output=True, + ) + memory_field, cpu_field = output.strip().split("\t") + used_memory = memory_field.split("/")[0].strip() + cpu_percent = float(cpu_field.strip().rstrip("%")) + return Sample( + rss=container_memory_to_bytes(used_memory), + cpu_percent=cpu_percent, + ) + except Exception: + return Sample() + + +def read_last_elapsed_s(log_path: Path) -> float | None: + """ + Read the `elapsed_s` of the last sample of an existing usage log, so + that a further run can continue from it; 0.0 if the log has a header + but no samples yet. None if the log does not start with a header row, + in which case it holds nothing worth keeping and can be overwritten. + """ + lines = log_path.read_text().splitlines() + if not lines or lines[0].split("\t")[0] != fields(Sample)[0].name: + return None + for line in reversed(lines[1:]): + elapsed_s = line.split("\t")[0] + if elapsed_s: + try: + return float(elapsed_s) + except ValueError: + continue + return 0.0 + + +class ResourceMonitor: + """ + Monitor resource usage (memory, CPU) of an index-building + process. Works in both native mode (via psutil) and container mode + (via docker/podman stats). + + Usage as a context manager: + + with ResourceMonitor(dataset="wikidata", binary="qlever-index"): + run_command(cmd, show_output=True) + + # For container mode: + with ResourceMonitor(dataset="wikidata", + binary="qlever-index", + container="qlever.index.wikidata", + system="docker"): + run_command(cmd, show_output=True) + """ + + def __init__( + self, + dataset: str, + binary: str, + container: str | None = None, + system: str | None = None, + interval: float = 1.0, + output_dir: Path | None = None, + parent_pid: int | None = None, + append: bool = False, + ): + """ + Args: + dataset: Name of the dataset being indexed. + binary: Name of the index executable, matched against the + descendant processes (native mode only). + container: Container name to sample; when set with `system`, + sampling uses `docker/podman stats` not psutil. + system: Container runtime ("docker" or "podman"). + interval: Seconds between samples. + output_dir: Directory for the TSV usage log file. + parent_pid: PID whose descendants are searched for the index + process. Defaults to the current process; pass a + different PID when the target re-parents away from + us. + append: Add this run's samples to an existing usage log + instead of overwriting it, continuing `elapsed_s` + from its last row. For engines that build an index + in several runs. A run that raises is rolled back. + """ + self.dataset = dataset + self.binary = binary + self.container = container + self.system = system + self.interval = interval + self.output_dir = output_dir or Path.cwd() + self.parent_pid = parent_pid + self.append = append + self.peak_rss = 0 + self.worker_proc = None + self.log_file = None + self.stop_event = threading.Event() + self.start_time = 0 + # Set in `__enter__` when appending to an existing log: the + # `elapsed_s` to continue from, and the size the file had before + # this run, which `__exit__` truncates back to on failure. + self.elapsed_offset = 0.0 + self.append_offset = None + + @classmethod + def from_args(cls, args) -> ResourceMonitor: + """Monitor the index build configured by `args`.""" + return cls( + dataset=args.name, + binary=args.index_binary, + container=args.index_container, + system=args.system, + interval=args.resource_usage_interval, + ) + + def take_sample(self) -> Sample: + """ + Dispatch to container or native sampling, caching the resolved + process. + """ + if self.system in Containerize.supported_systems(): + return sample_container(self.system, self.container) + if self.worker_proc is None or not self.worker_proc.is_running(): + self.worker_proc = find_process_by_binary( + self.parent_pid, self.binary + ) + if self.worker_proc is None: + return Sample() + # cpu_percent reports usage since the previous call, so this + # first call seeds the baseline and its 0.0 result is discarded. + try: + self.worker_proc.cpu_percent(interval=None) + except (psutil.NoSuchProcess, psutil.AccessDenied): + self.worker_proc = None + return Sample() + return sample_process(self.worker_proc) + + def run_loop(self): + """ + Polling loop on a background thread. Samples resource usage + and appends one TSV row per iteration until stop_event is set. + """ + while not self.stop_event.is_set(): + sample = self.take_sample() + sample.elapsed_s = round( + self.elapsed_offset + time.monotonic() - self.start_time, 1 + ) + if sample.rss is not None and self.log_file is not None: + self.peak_rss = max(self.peak_rss, sample.rss) + self.log_file.write(sample_to_tsv_row(sample)) + self.log_file.flush() + self.stop_event.wait(self.interval) + + def __enter__(self): + """ + Open the TSV log and start the sampling thread. Writes a header to + a fresh log; continues an existing one when `append` was set. + """ + self.log_path = ( + self.output_dir / f"{self.dataset}.index.resource-usage-log.tsv" + ) + previous_elapsed_s = ( + read_last_elapsed_s(self.log_path) + if self.append and self.log_path.exists() + else None + ) + if previous_elapsed_s is None: + self.log_file = open(self.log_path, "w") + header = "\t".join(f.name for f in fields(Sample)) + "\n" + self.log_file.write(header) + else: + self.elapsed_offset = previous_elapsed_s + self.append_offset = self.log_path.stat().st_size + self.log_file = open(self.log_path, "a") + self.log_file.flush() + self.start_time = time.monotonic() + self.thread = threading.Thread(target=self.run_loop, daemon=True) + self.thread.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Stop sampling and close the log, reporting where it was saved. + When appending, a run that raised is rolled back: it completed no + index build, so its samples belong to no run and would stretch + `elapsed_s` beyond what the index log accounts for. + """ + self.stop_event.set() + self.thread.join() + self.log_file.close() + if exc_type is not None and self.append_offset is not None: + with open(self.log_path, "r+") as log_file: + log_file.truncate(self.append_offset) + log.warning( + "Discarded the resource-usage samples of the failed run " + f"from `{self.log_path.name}`" + ) + return False + if self.peak_rss > 0: + log.info( + "Resource-usage log (RSS memory and CPU usage) saved to " + f"`{self.log_path.name}`" + ) + else: + log.warning( + "Resource usage was not recorded (no samples collected)." + ) + return False diff --git a/src/qlever/resource_usage/usage_plot.py b/src/qlever/resource_usage/usage_plot.py index a0510c7a..af9cad95 100644 --- a/src/qlever/resource_usage/usage_plot.py +++ b/src/qlever/resource_usage/usage_plot.py @@ -3,6 +3,7 @@ import csv import json import warnings +from collections.abc import Callable from datetime import datetime from pathlib import Path @@ -159,6 +160,48 @@ def add(name: str, start: datetime | None, end: datetime | None) -> None: return phases +def bands_from_durations( + durations: dict[str, float], +) -> list[tuple[str, float, float]]: + """ + Turn phase durations in seconds into `(label, start_s, end_s)` bands, + laying the phases back to back from the build start in the given + order. The `TOTAL time` entry is skipped. + """ + bands = [] + start_s = 0.0 + for label, duration_s in durations.items(): + if label == "TOTAL time": + continue + bands.append((label, start_s, start_s + duration_s)) + start_s += duration_s + return bands + + +# Separator between the fields of a subtitle, and the width at which the +# title starts to be clipped by a 12in figure's axes. +SUBTITLE_SEPARATOR = " | " +SUBTITLE_MAX_CHARS = 105 + + +def wrap_subtitle(subtitle: str) -> str: + """Break a subtitle at its field separators into lines that fit the axes.""" + lines = [] + for line in subtitle.split("\n"): + fields = line.split(SUBTITLE_SEPARATOR) + current = fields[0] + for field in fields[1:]: + if len(current) + len(SUBTITLE_SEPARATOR) + len(field) > ( + SUBTITLE_MAX_CHARS + ): + lines.append(current) + current = field + else: + current += SUBTITLE_SEPARATOR + field + lines.append(current) + return "\n".join(lines) + + def build_plot_subtitle( log_path: Path, stxxl_memory: str, settings_json: str ) -> str | None: @@ -183,24 +226,39 @@ def build_plot_subtitle( parts.append(f"git = {git_hash}") if stxxl_memory: parts.append(f"STXXL = {stxxl_memory}") - return " | ".join(parts) if parts else None + return SUBTITLE_SEPARATOR.join(parts) if parts else None + + +def qlever_overlay(args, log_path: Path) -> list[tuple[str, float, float]]: + """Shade one band per phase of a QLever index build.""" + phases = compute_phase_boundaries(log_path) + return [ + (name, start_s, end_s) for name, (start_s, end_s) in phases.items() + ] + + +def qlever_subtitle(args, log_path: Path) -> str | None: + """Subtitle for a QLever index build.""" + return build_plot_subtitle( + log_path, args.stxxl_memory or "", args.settings_json + ) def write_usage_plot( tsv_path: Path, - log_path: Path, - stxxl_memory: str, - settings_json: str, out_path: Path, title: str, + overlay: list[tuple[str, float, float]], + subtitle: str | None, plot_max_points: int = 500, + sample_interval_s: float = 1.0, ) -> bool: """ - Read the usage TSV and index log, render a dual-axis plot of - memory and CPU over time with phase bands from the index log, - and save it to `out_path`. Returns True if a plot was saved, - False if the TSV has no usable samples. `plot_max_points` caps - the number of points drawn per series. + Read the usage TSV, render a dual-axis plot of memory and CPU over + time with the `overlay` regions shaded, and save it to `out_path`. + Returns True if a plot was saved, False if the TSV has no usable + samples. `plot_max_points` caps the number of points drawn per + series. """ data = read_usage_tsv(tsv_path) if not data or len(data.get("elapsed_s", [])) == 0: @@ -214,8 +272,6 @@ def write_usage_plot( data = {name: values[valid[0] :] for name, values in data.items()} data["elapsed_s"] = data["elapsed_s"] - data["elapsed_s"][0] - phases = compute_phase_boundaries(log_path) - data = downsample_for_plot(data, plot_max_points) elapsed_s = data["elapsed_s"] @@ -232,10 +288,10 @@ def write_usage_plot( band_colors = plt.colormaps["Pastel1"].colors total_s = float(elapsed_s[-1]) if len(elapsed_s) else 0.0 - # skip drawing the phase name when the band is too narrow to fit it + # skip drawing the region name when the band is too narrow to fit it # legibly; arbitrary 2% of total duration. min_label_s = total_s * 0.02 - for band_idx, (name, (start_s, end_s)) in enumerate(phases.items()): + for band_idx, (name, start_s, end_s) in enumerate(overlay): band_s = end_s - start_s if band_s <= 0: continue @@ -248,9 +304,13 @@ def write_usage_plot( ) if band_s < min_label_s: continue - mid = (start_s + end_s) / 2 / x_factor + mid = (start_s + end_s) / 2 + # Skip the name when the band's middle is past the last sample. + # Text that far outside the axes collapses the layout. + if mid > total_s: + continue ax_mem.text( - mid, + mid / x_factor, 0.98, name, transform=ax_mem.get_xaxis_transform(), @@ -298,28 +358,47 @@ def write_usage_plot( bbox_to_anchor=(1.08, 0.5), ) - subtitle = build_plot_subtitle(log_path, stxxl_memory, settings_json) - ax_mem.set_title(f"{title}\n{subtitle}" if subtitle else title) + # The bands describe the index log's timeline, the axis describes the + # samples. If the bands reach past the last sample, the two do not + # cover the same run and the shading sits on the wrong part of the + # curve. Allow for the sampling stopping a little early. + tolerance_s = 2 * sample_interval_s + 5 + bands_end_s = max((end_s for _, _, end_s in overlay), default=0.0) + if bands_end_s > total_s + tolerance_s: + note = "(!) shading exceeds the sampled range" + log.warning( + f"The shaded regions cover {bands_end_s:.0f}s but only " + f"{total_s:.0f}s were sampled, so they may not line up with " + "the curves" + ) + # On its own line: the subtitle is already near the axes width. + subtitle = f"{subtitle}\n{note}" if subtitle else note + + ax_mem.set_title( + f"{title}\n{wrap_subtitle(subtitle)}" if subtitle else title + ) fig.savefig(out_path, dpi=120) plt.close(fig) return True def render_usage_plot( - dataset: str, - engine_display: str, - stxxl_memory: str = "", - settings_json: str = "{}", + args, + *, + overlay: Callable[..., list[tuple[str, float, float]]], + subtitle: Callable[..., str | None], output_dir: Path | None = None, - plot_max_points: int = 500, ) -> Path | None: """ - Render `.resource-usage-plot.png` from - `.index.resource-usage-log.tsv` in `output_dir`, falling - back to `.resource-usage-log.tsv` as written by older - qlever versions. Returns the plot path on success, None if the log - is missing or the plot could not be rendered. + Render `.resource-usage-plot.png` from + `.index.resource-usage-log.tsv` in `output_dir`, falling back + to `.resource-usage-log.tsv` as written by older qlever + versions. `overlay` and `subtitle` are called with `(args, + log_path)` and provide the engine-specific parts of the plot. + Returns the plot path on success, None if the log is missing or the + plot could not be rendered. """ + dataset = args.name output_dir = output_dir or Path.cwd() tsv_path = output_dir / f"{dataset}.index.resource-usage-log.tsv" # Backwards compatibility with older resource-usage log filename @@ -333,12 +412,12 @@ def render_usage_plot( try: rendered = write_usage_plot( tsv_path=tsv_path, - log_path=log_path, - stxxl_memory=stxxl_memory, - settings_json=settings_json, out_path=plot_path, - title=f"{engine_display} index build: {dataset}", - plot_max_points=plot_max_points, + title=f"{args.engine_display} index build: {dataset}", + overlay=overlay(args, log_path), + subtitle=subtitle(args, log_path), + plot_max_points=args.resource_usage_plot_max_points, + sample_interval_s=args.resource_usage_interval, ) except Exception as error: log.warning(f"Could not render resource-usage plot: {error}") diff --git a/test/qlever/resource_usage/test_resource_monitor.py b/test/qlever/resource_usage/test_resource_monitor.py new file mode 100644 index 00000000..ad94409f --- /dev/null +++ b/test/qlever/resource_usage/test_resource_monitor.py @@ -0,0 +1,59 @@ +from unittest.mock import MagicMock + +import psutil +import pytest + +from qlever.resource_usage.resource_monitor import ( + Sample, + sample_container, + sample_process, + sample_to_tsv_row, +) + +MODULE = "qlever.resource_usage.resource_monitor" + + +@pytest.mark.parametrize( + "sample,expected", + [ + (Sample(elapsed_s=1.0, rss=100, cpu_percent=5.0), "1.0\t100\t5.0\n"), + (Sample(), "\t\t\n"), + (Sample(elapsed_s=2.0), "2.0\t\t\n"), + # Zero is a real reading, not a missing one: it renders as "0" + # / "0.0", never as an empty column. + (Sample(elapsed_s=0.0, rss=0, cpu_percent=0.0), "0.0\t0\t0.0\n"), + ], +) +def test_sample_to_tsv_row(sample, expected): + assert sample_to_tsv_row(sample) == expected + + +def test_sample_container_parses_stats_output(mock_command): + run_cmd_mock = mock_command(MODULE, "run_command") + run_cmd_mock.return_value = "1.5GiB / 7.6GiB\t12.5%" + sample = sample_container("docker", "qlever.index.test") + assert sample.rss == int(1.5 * 1024**3) + assert sample.cpu_percent == 12.5 + + +def test_sample_container_returns_empty_on_malformed_output(mock_command): + run_cmd_mock = mock_command(MODULE, "run_command") + run_cmd_mock.return_value = "garbage" + sample = sample_container("docker", "qlever.index.test") + assert sample == Sample() + + +def test_sample_process_reads_rss_and_cpu(): + proc = MagicMock() + proc.memory_info.return_value.rss = 2048 + proc.cpu_percent.return_value = 7.5 + sample = sample_process(proc) + assert sample.rss == 2048 + assert sample.cpu_percent == 7.5 + + +def test_sample_process_returns_empty_when_process_gone(): + proc = MagicMock() + proc.memory_info.side_effect = psutil.NoSuchProcess(pid=123) + sample = sample_process(proc) + assert sample == Sample() diff --git a/test/qlever/resource_usage/test_usage_plot.py b/test/qlever/resource_usage/test_usage_plot.py index b581e897..68d28ad5 100644 --- a/test/qlever/resource_usage/test_usage_plot.py +++ b/test/qlever/resource_usage/test_usage_plot.py @@ -1,3 +1,6 @@ +import logging +from types import SimpleNamespace + import pytest # The plot extra (numpy, matplotlib) is optional, so skip this whole @@ -6,12 +9,18 @@ pytest.importorskip("matplotlib") from qlever.resource_usage.usage_plot import ( # noqa: E402 + SUBTITLE_SEPARATOR, + bands_from_durations, build_plot_subtitle, compute_phase_boundaries, downsample_for_plot, pick_time_unit, + qlever_overlay, + qlever_subtitle, read_usage_tsv, render_usage_plot, + wrap_subtitle, + write_usage_plot, ) @@ -209,8 +218,84 @@ def test_compute_phase_boundaries_skips_incomplete_phase(tmp_path): assert phases == {} +def test_bands_from_durations_lays_phases_back_to_back(): + bands = bands_from_durations( + {"Load": 10.0, "Optimize": 5.0, "TOTAL time": 15.0} + ) + assert bands == [("Load", 0.0, 10.0), ("Optimize", 10.0, 15.0)] + + +def test_wrap_subtitle_keeps_a_short_subtitle_on_one_line(): + subtitle = SUBTITLE_SEPARATOR.join(["batch = 10M triples", "git = abc123"]) + assert wrap_subtitle(subtitle) == subtitle + + +def test_wrap_subtitle_breaks_a_long_subtitle_at_the_separators(): + fields = [f"field {i} = {'x' * 20}" for i in range(5)] + lines = wrap_subtitle(SUBTITLE_SEPARATOR.join(fields)).split("\n") + assert len(lines) > 1 + assert all(len(line) <= 105 for line in lines) + # No field is split in the middle and none is lost. + assert SUBTITLE_SEPARATOR.join(lines).split(SUBTITLE_SEPARATOR) == fields + + +def write_samples(tmp_path, last_elapsed_s): + """Write a two-row usage TSV ending at `last_elapsed_s`.""" + tsv_path = tmp_path / "data.tsv" + tsv_path.write_text( + "elapsed_s\trss\tcpu_percent\n" + f"0\t100\t5.0\n{last_elapsed_s}\t200\t6.0\n" + ) + return tsv_path + + +def write_plot_with_overlay(tmp_path, last_elapsed_s, overlay): + """Render a plot from `overlay` over samples ending at `last_elapsed_s`.""" + return write_usage_plot( + tsv_path=write_samples(tmp_path, last_elapsed_s), + out_path=tmp_path / "plot.png", + title="Test", + overlay=overlay, + subtitle=None, + ) + + +def test_write_usage_plot_warns_when_shading_exceeds_samples(tmp_path, caplog): + with caplog.at_level(logging.WARNING, logger="qlever"): + assert write_plot_with_overlay(tmp_path, 10, [("Phase", 0.0, 300.0)]) + assert "300s" in caplog.text and "10s were sampled" in caplog.text + + +def test_write_usage_plot_quiet_when_shading_fits_samples(tmp_path, caplog): + with caplog.at_level(logging.WARNING, logger="qlever"): + assert write_plot_with_overlay(tmp_path, 10, [("Phase", 0.0, 10.0)]) + assert caplog.text == "" + + +def plot_args(name): + """The `args` attributes that `render_usage_plot` reads.""" + return SimpleNamespace( + name=name, + engine_display="QLever", + resource_usage_plot_max_points=500, + resource_usage_interval=1, + stxxl_memory="", + settings_json="{}", + ) + + +def render(name, tmp_path): + """Render a QLever usage plot for `name` in `tmp_path`.""" + return render_usage_plot( + plot_args(name), + overlay=qlever_overlay, + subtitle=qlever_subtitle, + output_dir=tmp_path, + ) + + def test_render_usage_plot_missing_tsv(tmp_path): - assert render_usage_plot("missing", "QLever", output_dir=tmp_path) is None + assert render("missing", tmp_path) is None def test_render_usage_plot_header_only_tsv_renders_nothing(tmp_path): @@ -218,7 +303,7 @@ def test_render_usage_plot_header_only_tsv_renders_nothing(tmp_path): # or leave a PNG behind. tsv_path = tmp_path / "data.index.resource-usage-log.tsv" tsv_path.write_text("elapsed_s\trss\tcpu_percent\n") - assert render_usage_plot("data", "QLever", output_dir=tmp_path) is None + assert render("data", tmp_path) is None assert not (tmp_path / "data.resource-usage-plot.png").exists() @@ -228,6 +313,6 @@ def test_render_usage_plot_falls_back_to_old_tsv_name(tmp_path): tsv_path.write_text( "elapsed_s\trss\tcpu_percent\n1.0\t100\t5.0\n2.0\t200\t6.0\n" ) - plot_path = render_usage_plot("data", "QLever", output_dir=tmp_path) + plot_path = render("data", tmp_path) assert plot_path == tmp_path / "data.resource-usage-plot.png" assert plot_path.exists() From cbd94dca43e1f2de543561dc750efeb5e6a9881f Mon Sep 17 00:00:00 2001 From: tanmay-9 Date: Tue, 18 Aug 2026 13:14:25 +0200 Subject: [PATCH 6/7] Add . prefix to resource-usage log and plot for non-qlever engines --- src/qlever/commands/index.py | 31 +++++++---- src/qlever/resource_usage/resource_monitor.py | 16 ++++-- src/qlever/resource_usage/usage_plot.py | 54 ++++++++++--------- src/qlever/util.py | 11 ++++ test/qlever/resource_usage/test_usage_plot.py | 21 ++++---- 5 files changed, 86 insertions(+), 47 deletions(-) diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index 695c776a..eeef88b9 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -4,6 +4,7 @@ import json import re import shlex +from importlib import import_module from pathlib import Path from qlever.command import QleverCommand @@ -17,15 +18,23 @@ run_command, ) +USAGE_PLOT_MODULE = "qlever.resource_usage.usage_plot" -def render_usage_plot(args, plot_only: bool) -> Path | None: - """Render the resource-usage plot. - When the plotting libraries are missing, this is an error if the - user asked for the plot directly via `plot_only`, otherwise it notes - how to get the plot at info level since the index build succeeded. +def render_usage_plot( + args, plot_only: bool, engine_module: str +) -> Path | None: + """ + Render the resource-usage plot, taking the engine-specific parts from + the `overlay` and `subtitle` of `engine_module`. + + The plotting libraries are an optional dependency, so the import has + to happen here and not at module level. Missing them is an error when + the user asked for the plot directly via `plot_only`, and only a hint + otherwise, because then the index build itself succeeded. """ try: + engine_plot = import_module(engine_module) from qlever.resource_usage import usage_plot except ImportError: if plot_only: @@ -42,8 +51,8 @@ def render_usage_plot(args, plot_only: bool) -> Path | None: return None return usage_plot.render_usage_plot( args, - overlay=usage_plot.qlever_overlay, - subtitle=usage_plot.qlever_subtitle, + engine_overlay=engine_plot.overlay, + engine_subtitle=engine_plot.subtitle, ) @@ -226,7 +235,9 @@ def execute(self, args) -> bool: # Render the resource-usage plot from the existing log without # rebuilding the index. if args.resource_usage_plot_only: - plot_path = render_usage_plot(args, plot_only=True) + plot_path = render_usage_plot( + args, plot_only=True, engine_module=USAGE_PLOT_MODULE + ) if plot_path is None: return False log.info(f"Resource-usage plot saved to `{plot_path.name}`") @@ -397,7 +408,9 @@ def execute(self, args) -> bool: Path(f"{args.name}.index.resource-usage-log.tsv").exists() or Path(f"{args.name}.resource-usage-log.tsv").exists() ): - plot_path = render_usage_plot(args, plot_only=False) + plot_path = render_usage_plot( + args, plot_only=False, engine_module=USAGE_PLOT_MODULE + ) if plot_path is not None: log.info(f"Resource-usage plot saved to `{plot_path.name}`") diff --git a/src/qlever/resource_usage/resource_monitor.py b/src/qlever/resource_usage/resource_monitor.py index 5735cfbc..ec53de3d 100644 --- a/src/qlever/resource_usage/resource_monitor.py +++ b/src/qlever/resource_usage/resource_monitor.py @@ -12,6 +12,7 @@ from qlever.util import ( container_memory_to_bytes, find_process_by_binary, + resource_usage_prefix, run_command, ) @@ -94,13 +95,15 @@ class ResourceMonitor: Usage as a context manager: - with ResourceMonitor(dataset="wikidata", binary="qlever-index"): + with ResourceMonitor(dataset="wikidata", engine="oxigraph", + binary="oxigraph"): run_command(cmd, show_output=True) # For container mode: with ResourceMonitor(dataset="wikidata", - binary="qlever-index", - container="qlever.index.wikidata", + engine="oxigraph", + binary="oxigraph", + container="oxigraph.index.wikidata", system="docker"): run_command(cmd, show_output=True) """ @@ -108,6 +111,7 @@ class ResourceMonitor: def __init__( self, dataset: str, + engine: str, binary: str, container: str | None = None, system: str | None = None, @@ -119,6 +123,7 @@ def __init__( """ Args: dataset: Name of the dataset being indexed. + engine: Engine key, which the log and plot names start with. binary: Name of the index executable, matched against the descendant processes (native mode only). container: Container name to sample; when set with `system`, @@ -136,6 +141,7 @@ def __init__( in several runs. A run that raises is rolled back. """ self.dataset = dataset + self.engine = engine self.binary = binary self.container = container self.system = system @@ -159,6 +165,7 @@ def from_args(cls, args) -> ResourceMonitor: """Monitor the index build configured by `args`.""" return cls( dataset=args.name, + engine=args.engine, binary=args.index_binary, container=args.index_container, system=args.system, @@ -208,8 +215,9 @@ def __enter__(self): Open the TSV log and start the sampling thread. Writes a header to a fresh log; continues an existing one when `append` was set. """ + prefix = resource_usage_prefix(self.engine, self.dataset) self.log_path = ( - self.output_dir / f"{self.dataset}.index.resource-usage-log.tsv" + self.output_dir / f"{prefix}.index.resource-usage-log.tsv" ) previous_elapsed_s = ( read_last_elapsed_s(self.log_path) diff --git a/src/qlever/resource_usage/usage_plot.py b/src/qlever/resource_usage/usage_plot.py index af9cad95..0252cd74 100644 --- a/src/qlever/resource_usage/usage_plot.py +++ b/src/qlever/resource_usage/usage_plot.py @@ -19,10 +19,14 @@ iter_permutation_phases, parse_git_hash, parse_phase_markers, + resource_usage_prefix, ) GB = 1024**3 +# One shaded region of the plot: name, start and end in seconds. +BandType = tuple[str, float, float] + def read_usage_tsv(path: Path) -> dict[str, np.ndarray]: """ @@ -162,7 +166,7 @@ def add(name: str, start: datetime | None, end: datetime | None) -> None: def bands_from_durations( durations: dict[str, float], -) -> list[tuple[str, float, float]]: +) -> list[BandType]: """ Turn phase durations in seconds into `(label, start_s, end_s)` bands, laying the phases back to back from the build start in the given @@ -184,10 +188,10 @@ def bands_from_durations( SUBTITLE_MAX_CHARS = 105 -def wrap_subtitle(subtitle: str) -> str: +def wrap_subtitle(text: str) -> str: """Break a subtitle at its field separators into lines that fit the axes.""" lines = [] - for line in subtitle.split("\n"): + for line in text.split("\n"): fields = line.split(SUBTITLE_SEPARATOR) current = fields[0] for field in fields[1:]: @@ -229,7 +233,7 @@ def build_plot_subtitle( return SUBTITLE_SEPARATOR.join(parts) if parts else None -def qlever_overlay(args, log_path: Path) -> list[tuple[str, float, float]]: +def overlay(args, log_path: Path) -> list[BandType]: """Shade one band per phase of a QLever index build.""" phases = compute_phase_boundaries(log_path) return [ @@ -237,7 +241,7 @@ def qlever_overlay(args, log_path: Path) -> list[tuple[str, float, float]]: ] -def qlever_subtitle(args, log_path: Path) -> str | None: +def subtitle(args, log_path: Path) -> str | None: """Subtitle for a QLever index build.""" return build_plot_subtitle( log_path, args.stxxl_memory or "", args.settings_json @@ -248,14 +252,14 @@ def write_usage_plot( tsv_path: Path, out_path: Path, title: str, - overlay: list[tuple[str, float, float]], - subtitle: str | None, + bands: list[BandType], + subtitle_text: str | None, plot_max_points: int = 500, sample_interval_s: float = 1.0, ) -> bool: """ Read the usage TSV, render a dual-axis plot of memory and CPU over - time with the `overlay` regions shaded, and save it to `out_path`. + time with the `bands` regions shaded, and save it to `out_path`. Returns True if a plot was saved, False if the TSV has no usable samples. `plot_max_points` caps the number of points drawn per series. @@ -291,7 +295,7 @@ def write_usage_plot( # skip drawing the region name when the band is too narrow to fit it # legibly; arbitrary 2% of total duration. min_label_s = total_s * 0.02 - for band_idx, (name, start_s, end_s) in enumerate(overlay): + for band_idx, (name, start_s, end_s) in enumerate(bands): band_s = end_s - start_s if band_s <= 0: continue @@ -363,7 +367,7 @@ def write_usage_plot( # cover the same run and the shading sits on the wrong part of the # curve. Allow for the sampling stopping a little early. tolerance_s = 2 * sample_interval_s + 5 - bands_end_s = max((end_s for _, _, end_s in overlay), default=0.0) + bands_end_s = max((end_s for _, _, end_s in bands), default=0.0) if bands_end_s > total_s + tolerance_s: note = "(!) shading exceeds the sampled range" log.warning( @@ -372,10 +376,10 @@ def write_usage_plot( "the curves" ) # On its own line: the subtitle is already near the axes width. - subtitle = f"{subtitle}\n{note}" if subtitle else note + subtitle_text = f"{subtitle_text}\n{note}" if subtitle_text else note ax_mem.set_title( - f"{title}\n{wrap_subtitle(subtitle)}" if subtitle else title + f"{title}\n{wrap_subtitle(subtitle_text)}" if subtitle_text else title ) fig.savefig(out_path, dpi=120) plt.close(fig) @@ -385,27 +389,29 @@ def write_usage_plot( def render_usage_plot( args, *, - overlay: Callable[..., list[tuple[str, float, float]]], - subtitle: Callable[..., str | None], + engine_overlay: Callable[..., list[BandType]], + engine_subtitle: Callable[..., str | None], output_dir: Path | None = None, ) -> Path | None: """ - Render `.resource-usage-plot.png` from - `.index.resource-usage-log.tsv` in `output_dir`, falling back - to `.resource-usage-log.tsv` as written by older qlever - versions. `overlay` and `subtitle` are called with `(args, - log_path)` and provide the engine-specific parts of the plot. + Render `.resource-usage-plot.png` from + `.index.resource-usage-log.tsv` in `output_dir`, where + `prefix` comes from `resource_usage_prefix`, falling back to + `.resource-usage-log.tsv` as written by older qlever + versions. `engine_overlay` and `engine_subtitle` are called with + `(args, log_path)` and provide the engine-specific parts of the plot. Returns the plot path on success, None if the log is missing or the plot could not be rendered. """ dataset = args.name + prefix = resource_usage_prefix(args.engine, dataset) output_dir = output_dir or Path.cwd() - tsv_path = output_dir / f"{dataset}.index.resource-usage-log.tsv" + tsv_path = output_dir / f"{prefix}.index.resource-usage-log.tsv" # Backwards compatibility with older resource-usage log filename if not tsv_path.exists(): - tsv_path = output_dir / f"{dataset}.resource-usage-log.tsv" + tsv_path = output_dir / f"{prefix}.resource-usage-log.tsv" log_path = output_dir / f"{dataset}.index-log.txt" - plot_path = output_dir / f"{dataset}.resource-usage-plot.png" + plot_path = output_dir / f"{prefix}.resource-usage-plot.png" if not tsv_path.exists(): log.warning(f"Resource-usage log not found: `{tsv_path.name}`") return None @@ -414,8 +420,8 @@ def render_usage_plot( tsv_path=tsv_path, out_path=plot_path, title=f"{args.engine_display} index build: {dataset}", - overlay=overlay(args, log_path), - subtitle=subtitle(args, log_path), + bands=engine_overlay(args, log_path), + subtitle_text=engine_subtitle(args, log_path), plot_max_points=args.resource_usage_plot_max_points, sample_interval_s=args.resource_usage_interval, ) diff --git a/src/qlever/util.py b/src/qlever/util.py index d14c2fc2..b3a17f9f 100644 --- a/src/qlever/util.py +++ b/src/qlever/util.py @@ -259,6 +259,17 @@ def get_existing_index_files( return [path.name for path in existing_index_files] +def resource_usage_prefix(engine: str, dataset: str) -> str: + """ + Name that the resource-usage log and plot start with, that is + `.`. The engine goes after the dataset, so that it + is clear which engine a log or plot belongs to. QLever's own binaries + write the log and do not know the engine name, so QLever gets plain + ``. + """ + return dataset if engine == "qlever" else f"{dataset}.{engine}" + + def show_process_info(psutil_process, cmdline_regex, show_heading=True): """ Helper function that shows information about a process if information diff --git a/test/qlever/resource_usage/test_usage_plot.py b/test/qlever/resource_usage/test_usage_plot.py index 68d28ad5..89def1fc 100644 --- a/test/qlever/resource_usage/test_usage_plot.py +++ b/test/qlever/resource_usage/test_usage_plot.py @@ -14,11 +14,11 @@ build_plot_subtitle, compute_phase_boundaries, downsample_for_plot, + overlay, pick_time_unit, - qlever_overlay, - qlever_subtitle, read_usage_tsv, render_usage_plot, + subtitle, wrap_subtitle, write_usage_plot, ) @@ -249,26 +249,26 @@ def write_samples(tmp_path, last_elapsed_s): return tsv_path -def write_plot_with_overlay(tmp_path, last_elapsed_s, overlay): - """Render a plot from `overlay` over samples ending at `last_elapsed_s`.""" +def write_plot_with_bands(tmp_path, last_elapsed_s, bands): + """Render a plot from `bands` over samples ending at `last_elapsed_s`.""" return write_usage_plot( tsv_path=write_samples(tmp_path, last_elapsed_s), out_path=tmp_path / "plot.png", title="Test", - overlay=overlay, - subtitle=None, + bands=bands, + subtitle_text=None, ) def test_write_usage_plot_warns_when_shading_exceeds_samples(tmp_path, caplog): with caplog.at_level(logging.WARNING, logger="qlever"): - assert write_plot_with_overlay(tmp_path, 10, [("Phase", 0.0, 300.0)]) + assert write_plot_with_bands(tmp_path, 10, [("Phase", 0.0, 300.0)]) assert "300s" in caplog.text and "10s were sampled" in caplog.text def test_write_usage_plot_quiet_when_shading_fits_samples(tmp_path, caplog): with caplog.at_level(logging.WARNING, logger="qlever"): - assert write_plot_with_overlay(tmp_path, 10, [("Phase", 0.0, 10.0)]) + assert write_plot_with_bands(tmp_path, 10, [("Phase", 0.0, 10.0)]) assert caplog.text == "" @@ -276,6 +276,7 @@ def plot_args(name): """The `args` attributes that `render_usage_plot` reads.""" return SimpleNamespace( name=name, + engine="qlever", engine_display="QLever", resource_usage_plot_max_points=500, resource_usage_interval=1, @@ -288,8 +289,8 @@ def render(name, tmp_path): """Render a QLever usage plot for `name` in `tmp_path`.""" return render_usage_plot( plot_args(name), - overlay=qlever_overlay, - subtitle=qlever_subtitle, + engine_overlay=overlay, + engine_subtitle=subtitle, output_dir=tmp_path, ) From f0e690273c47dd616218cc264fccba3a806d99d2 Mon Sep 17 00:00:00 2001 From: tanmay-9 Date: Tue, 18 Aug 2026 15:01:43 +0200 Subject: [PATCH 7/7] Fix failing test --- src/qlever/commands/start.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/qlever/commands/start.py b/src/qlever/commands/start.py index 71721148..730cdf6b 100644 --- a/src/qlever/commands/start.py +++ b/src/qlever/commands/start.py @@ -167,7 +167,8 @@ def get_runtime_parameters_from_qleverfile(args) -> list[str]: qleverfile_path = Path(vars(args).get("qleverfile", "Qleverfile")) if not qleverfile_path.is_file(): return [] - config = Qleverfile.read(qleverfile_path) + # The engine only names default containers, which we don't read here. + config = Qleverfile.read(qleverfile_path, vars(args).get("engine", "")) value = config.get("server", "set_runtime_parameters", fallback=None) return shlex.split(value) if value else [] except Exception: