From bcbcf4e8e5f2a7c99a9e3b757299527d9dbe9cb2 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Wed, 26 Aug 2026 00:07:51 +0200 Subject: [PATCH 1/4] Add option `--encode-as-id-wide` This complements ad-freiburg/qlever#3278. In the Qleverfile, write `ENCODE_AS_ID_WIDE`. --- src/qlever/commands/index.py | 3 +++ src/qlever/qleverfile.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index a9724eae..2a92c73a 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -76,6 +76,7 @@ def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: "input_files", "cat_input_files", "encode_as_id", + "encode_as_id_wide", "multi_input_json", "parallel_parsing", "settings_json", @@ -284,6 +285,8 @@ def execute(self, args) -> bool: # Add remaining options. if args.encode_as_id: index_cmd += f" --encode-as-id {args.encode_as_id}" + if args.encode_as_id_wide: + index_cmd += f" --encode-as-id-wide {args.encode_as_id_wide}" if args.only_pso_and_pos_permutations: index_cmd += " --only-pso-and-pos-permutations" if args.use_patterns == "no": diff --git a/src/qlever/qleverfile.py b/src/qlever/qleverfile.py index a795b301..150bfc35 100644 --- a/src/qlever/qleverfile.py +++ b/src/qlever/qleverfile.py @@ -205,6 +205,13 @@ def arg(*args, **kwargs): "by a sequence of digits, do not require a vocabulary entry but " "are directly encoded in the ID (default: none)", ) + index_args["encode_as_id_wide"] = arg( + "--encode-as-id-wide", + type=str, + help="Like --encode-as-id, but for IRIs with up to 17 digits " + "(at most 4 prefixes; the encoded IDs are ordered numerically, " + "not lexicographically; default: none)", + ) index_args["only_pso_and_pos_permutations"] = arg( "--only-pso-and-pos-permutations", action="store_true", From b252e51f9682df271ea335028bc7ad5c09018fa6 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Wed, 26 Aug 2026 00:19:18 +0200 Subject: [PATCH 2/4] Fix the index command tests for `--encode-as-id-wide` The new option was missing from the expected argument list and from the mocked arguments, so three tests saw a `MagicMock` instead of `None` and passed it on to the index command. --- test/qlever/commands/test_index_execute.py | 3 +++ test/qlever/commands/test_index_other_methods.py | 1 + 2 files changed, 4 insertions(+) diff --git a/test/qlever/commands/test_index_execute.py b/test/qlever/commands/test_index_execute.py index 525f4287..535d5d62 100644 --- a/test/qlever/commands/test_index_execute.py +++ b/test/qlever/commands/test_index_execute.py @@ -57,6 +57,7 @@ def test_execute_successful_indexing_without_extras( args.multi_input_json = False args.ulimit = None args.encode_as_id = None + args.encode_as_id_wide = None args.parser_buffer_size = None args.materialized_views = None @@ -273,6 +274,7 @@ def test_execute_total_file_size_greater_than_ten_gb( args.multi_input_json = False args.ulimit = None args.encode_as_id = None + args.encode_as_id_wide = None args.parser_buffer_size = None args.materialized_views = None @@ -386,6 +388,7 @@ def test_execute_successful_indexing_with_extras_and_show( args.show = True args.ulimit = None args.encode_as_id = None + args.encode_as_id_wide = None args.parser_buffer_size = None args.materialized_views = None diff --git a/test/qlever/commands/test_index_other_methods.py b/test/qlever/commands/test_index_other_methods.py index ff265020..fb6282e6 100644 --- a/test/qlever/commands/test_index_other_methods.py +++ b/test/qlever/commands/test_index_other_methods.py @@ -34,6 +34,7 @@ def test_relevant_qleverfile_arguments(self): "input_files", "cat_input_files", "encode_as_id", + "encode_as_id_wide", "multi_input_json", "parallel_parsing", "settings_json", From 54cec231b24127094097e95483f72d3c5524391e Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Thu, 27 Aug 2026 04:35:19 +0200 Subject: [PATCH 3/4] Add option `--parse-parallelism` This complements the corresponding option of the QLever index builder, which controls the number of threads for the first pass of the index build (parsing and converting triples to IDs). In the Qleverfile, write `PARSE_PARALLELISM`. --- src/qlever/commands/index.py | 3 +++ src/qlever/qleverfile.py | 8 ++++++++ test/qlever/commands/test_index_execute.py | 3 +++ test/qlever/commands/test_index_other_methods.py | 1 + 4 files changed, 15 insertions(+) diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index 0e90cc77..617e0d14 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -79,6 +79,7 @@ def relevant_qleverfile_arguments(self) -> dict[str, list[str]]: "cat_input_files", "encode_as_id", "encode_as_id_wide", + "parse_parallelism", "multi_input_json", "parallel_parsing", "settings_json", @@ -295,6 +296,8 @@ def execute(self, args) -> bool: index_cmd += f" --encode-as-id {args.encode_as_id}" if args.encode_as_id_wide: index_cmd += f" --encode-as-id-wide {args.encode_as_id_wide}" + if args.parse_parallelism: + index_cmd += f" --parse-parallelism {args.parse_parallelism}" if args.only_pso_and_pos_permutations: index_cmd += " --only-pso-and-pos-permutations" if args.use_patterns == "no": diff --git a/src/qlever/qleverfile.py b/src/qlever/qleverfile.py index f060d6ea..fa2e967e 100644 --- a/src/qlever/qleverfile.py +++ b/src/qlever/qleverfile.py @@ -236,6 +236,14 @@ def arg(*args, **kwargs): "(at most 4 prefixes; the encoded IDs are ordered numerically, " "not lexicographically; default: none)", ) + index_args["parse_parallelism"] = arg( + "--parse-parallelism", + type=int, + help="The number of threads used for parsing the input and for " + "converting the parsed triples to IDs (the first pass of the " + "index build); increase on machines with many cores " + "(default: 8 parser threads and 10 converter threads)", + ) index_args["only_pso_and_pos_permutations"] = arg( "--only-pso-and-pos-permutations", action="store_true", diff --git a/test/qlever/commands/test_index_execute.py b/test/qlever/commands/test_index_execute.py index 05a29b97..179f6b42 100644 --- a/test/qlever/commands/test_index_execute.py +++ b/test/qlever/commands/test_index_execute.py @@ -49,6 +49,7 @@ def test_execute_successful_indexing_without_extras( args.ulimit = None args.encode_as_id = None args.encode_as_id_wide = None + args.parse_parallelism = None args.parser_buffer_size = None args.materialized_views = None args.resource_usage_log = "yes" @@ -267,6 +268,7 @@ def test_execute_total_file_size_greater_than_ten_gb( args.ulimit = None args.encode_as_id = None args.encode_as_id_wide = None + args.parse_parallelism = None args.parser_buffer_size = None args.materialized_views = None args.resource_usage_log = "yes" @@ -384,6 +386,7 @@ def test_execute_successful_indexing_with_extras_and_show( args.ulimit = None args.encode_as_id = None args.encode_as_id_wide = None + args.parse_parallelism = None args.parser_buffer_size = None args.materialized_views = None args.resource_usage_log = "yes" diff --git a/test/qlever/commands/test_index_other_methods.py b/test/qlever/commands/test_index_other_methods.py index 198a3430..25ddb4a6 100644 --- a/test/qlever/commands/test_index_other_methods.py +++ b/test/qlever/commands/test_index_other_methods.py @@ -35,6 +35,7 @@ def test_relevant_qleverfile_arguments(self): "cat_input_files", "encode_as_id", "encode_as_id_wide", + "parse_parallelism", "multi_input_json", "parallel_parsing", "settings_json", From d19f66042f7d0b0f3b906705a632522552409e04 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Fri, 28 Aug 2026 03:18:18 +0200 Subject: [PATCH 4/4] Order `for-each` input files by size, largest first With parallel parsing, the largest inputs then start first, which minimizes the straggler tail at the end of the parsing phase. The order of the inputs is semantically irrelevant otherwise. --- src/qlever/commands/index.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index 617e0d14..3117c3b7 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -1,6 +1,7 @@ from __future__ import annotations import glob +import os import json import re import shlex @@ -191,7 +192,14 @@ def get_input_options_for_json(self, args) -> str: input_cmds = [input_spec["cmd"]] else: try: - files = sorted(glob.glob(input_spec["for-each"])) + # Sort the files by size, largest first: with parallel + # parsing, the largest inputs then start first, which + # minimizes the straggler tail at the end of the parsing + # phase (the order is semantically irrelevant otherwise). + files = sorted( + glob.glob(input_spec["for-each"]), + key=lambda f: (-os.path.getsize(f), f), + ) except Exception as e: raise self.InvalidInputJson( f"Element {i} in `MULTI_INPUT_JSON` contains an "