Add option --parse-parallelism

src/qlever/commands/index.py

@@ -1,21 +1,22 @@
 from __future__ import annotations
 
 import glob
+import os
 import json
 import re
 import shlex
 from pathlib import Path
 
 from qlever.command import QleverCommand
 from qlever.containerize import Containerize
 from qlever.log import log
 from qlever.util import (
     binary_exists,
     get_existing_index_files,
     get_total_file_size,
     input_files_exist,
     run_command,
 )
 
 
 def render_usage_plot(
@@ -78,6 +79,8 @@
                 "input_files",
                 "cat_input_files",
                 "encode_as_id",
+                "encode_as_id_wide",
+                "parse_parallelism",
                 "multi_input_json",
                 "parallel_parsing",
                 "settings_json",
@@ -189,7 +192,14 @@
                 input_cmds = [input_spec["cmd"]]
             else:
                 try:
-                    files = sorted(glob.glob(input_spec["for-each"]))
+                    # Sort the files by size, largest first: with parallel
+                    # parsing, the largest inputs then start first, which
+                    # minimizes the straggler tail at the end of the parsing
+                    # phase (the order is semantically irrelevant otherwise).
+                    files = sorted(
+                        glob.glob(input_spec["for-each"]),
+                        key=lambda f: (-os.path.getsize(f), f),
+                    )
                 except Exception as e:
                     raise self.InvalidInputJson(
                         f"Element {i} in `MULTI_INPUT_JSON` contains an "
@@ -292,6 +302,10 @@
         # Add remaining options.
         if args.encode_as_id:
             index_cmd += f" --encode-as-id {args.encode_as_id}"
+        if args.encode_as_id_wide:
+            index_cmd += f" --encode-as-id-wide {args.encode_as_id_wide}"
+        if args.parse_parallelism:
+            index_cmd += f" --parse-parallelism {args.parse_parallelism}"
         if args.only_pso_and_pos_permutations:
             index_cmd += " --only-pso-and-pos-permutations"
         if args.use_patterns == "no":

src/qlever/qleverfile.py

@@ -229,6 +229,21 @@ def arg(*args, **kwargs):
             "by a sequence of digits, do not require a vocabulary entry but "
             "are directly encoded in the ID (default: none)",
         )
+        index_args["encode_as_id_wide"] = arg(
+            "--encode-as-id-wide",
+            type=str,
+            help="Like --encode-as-id, but for IRIs with up to 17 digits "
+            "(at most 4 prefixes; the encoded IDs are ordered numerically, "
+            "not lexicographically; default: none)",
+        )
+        index_args["parse_parallelism"] = arg(
+            "--parse-parallelism",
+            type=int,
+            help="The number of threads used for parsing the input and for "
+            "converting the parsed triples to IDs (the first pass of the "
+            "index build); increase on machines with many cores "
+            "(default: 8 parser threads and 10 converter threads)",
+        )
         index_args["only_pso_and_pos_permutations"] = arg(
             "--only-pso-and-pos-permutations",
             action="store_true",

test/qlever/commands/test_index_execute.py

@@ -48,6 +48,8 @@ def test_execute_successful_indexing_without_extras(
         args.multi_input_json = False
         args.ulimit = None
         args.encode_as_id = None
+        args.encode_as_id_wide = None
+        args.parse_parallelism = None
         args.parser_buffer_size = None
         args.materialized_views = None
         args.resource_usage_log = "yes"
@@ -265,6 +267,8 @@ def test_execute_total_file_size_greater_than_ten_gb(
         args.multi_input_json = False
         args.ulimit = None
         args.encode_as_id = None
+        args.encode_as_id_wide = None
+        args.parse_parallelism = None
         args.parser_buffer_size = None
         args.materialized_views = None
         args.resource_usage_log = "yes"
@@ -381,6 +385,8 @@ def test_execute_successful_indexing_with_extras_and_show(
         args.show = True
         args.ulimit = None
         args.encode_as_id = None
+        args.encode_as_id_wide = None
+        args.parse_parallelism = None
         args.parser_buffer_size = None
         args.materialized_views = None
         args.resource_usage_log = "yes"

test/qlever/commands/test_index_other_methods.py

@@ -34,6 +34,8 @@ def test_relevant_qleverfile_arguments(self):
                     "input_files",
                     "cat_input_files",
                     "encode_as_id",
+                    "encode_as_id_wide",
+                    "parse_parallelism",
                     "multi_input_json",
                     "parallel_parsing",
                     "settings_json",