feat: env-var consolidation, score aggregation, and ManPage env-var documentation

Rules.md

@@ -54,6 +54,10 @@ The `mlpstorage` tool must be used to run the benchmarks, submitters are not all
 
 1.1. **mlpstorageGeneratesHierarchy** -- The `mlpstorage` command must obtain (somehow) the pathname of the output file directory hierarchy and directly create and/or append to the files within that hierarchy to successively build out the submission folder.  We don't want the submitter to manually create anything in that hierarchy except for the SystemDescription.* files (if we can help it).
 
+1.2. **storageBackendEnvVarDeclaration** -- Submitters MUST NOT set any documented s3dlio-side env var (§ENVIRONMENT → Storage-backend in ManPage.md) during the timed run unless the run's configuration file explicitly declares it. Results from runs with undeclared storage-backend env vars MUST be submitted as OPEN category. mlpstorage warns at run start when HIGH-risk vars are detected.
+
+For cloud-specific credential and endpoint env vars consumed by s3dlio for Azure or GCS backends, see s3dlio's [Environment_Variables.md](https://github.com/mlcommons/s3dlio/blob/main/docs/Environment_Variables.md).
+
 # 2. Core/Common Rules for All Submissions
 
 ## 2.1. Core/Common POSIX API Rules
@@ -666,7 +670,7 @@ is better), and the **aggregate Storage Throughput in tokens/s**.
 A run is launched with `mlpstorage <closed|open|whatif> kvcache run [OPTIONS]`
 (the division is the first positional, post-PR #412 modal CLI). The results
 directory must first be initialised once with `mlpstorage init <orgname>
-<results-dir>`, and every run requires `--systemname` (or the `MLPERF_SYSTEMNAME`
+<results-dir>`, and every run requires `--systemname` (or the `MLPSTORAGE_SYSTEMNAME`
 environment variable). The command executes **all three Options sequentially**,
 each repeated `trials` times, by prefixing `mpirun` to `mlperf_wrapper.py`. The
 Option's parameters are built by `mlpstorage` (`_build_option_kvcache_args`) and

mlpstorage_py/benchmarks/base.py

@@ -46,7 +46,13 @@ def _run(self):
 
 from functools import wraps
 
-from mlpstorage_py.config import PARAM_VALIDATION, DATETIME_STR, MLPS_DEBUG, EXEC_TYPE
+from mlpstorage_py.config import (
+    PARAM_VALIDATION,
+    DATETIME_STR,
+    MLPS_DEBUG,
+    EXEC_TYPE,
+    _S3DLIO_HIGH_RISK_ENV_VARS,
+)
 from mlpstorage_py.errors import ConfigurationError, ErrorCode, FileSystemError
 from mlpstorage_py.run_directory import (
     DEFAULT_COLLISION_BUMP_BUDGET,
@@ -991,6 +997,30 @@ def _fs_separation_paths(self) -> Optional[tuple]:
             f"_fs_separation_paths() for CAP-03"
         )
 
+    def _check_storage_backend_env(self) -> None:
+        """Warn when HIGH-risk s3dlio env vars are set (Phase 7.5 D-07, D-08).
+
+        Checks ``os.environ`` for entries in ``_S3DLIO_HIGH_RISK_ENV_VARS``
+        and emits a WARNING for each one found, but only when the active
+        storage library is s3dlio (``STORAGE_LIBRARY == 's3dlio'``).
+
+        Rationale for the STORAGE_LIBRARY guard (D-07): training-only
+        deployments that never use s3dlio should not see s3dlio warnings.
+        The guard avoids false positives without requiring per-var allowlists.
+
+        The warning uses the D-08 verbatim template so that automated tests
+        (test_d08_warning_prefix_in_benchmark_source) can lock the wording.
+        """
+        if os.environ.get('STORAGE_LIBRARY') != 's3dlio':
+            return
+        for var_name in sorted(_S3DLIO_HIGH_RISK_ENV_VARS):
+            if var_name in os.environ:
+                self.logger.warning(
+                    f"s3dlio env var '{var_name}' is set — this may alter measured benchmark results. "
+                    "Declare it in your config or submit as OPEN category. "
+                    "See ManPage §ENVIRONMENT → Storage-backend for impact."
+                )
+
     def _pre_execution_gate(self) -> None:
         """Run all pre-execution capacity/environment gates (CAP-01 in
         Slice 3; CAP-02 shared-FS verification is appended in Slice 4 of
@@ -1066,6 +1096,8 @@ def _pre_execution_gate(self) -> None:
         # design decision D-601-1). --skip-fs-separation-gate bypasses the
         # raise but still writes the sidecar so the validator gets telemetry.
         self._run_fs_separation_probe()
+        # Phase 7.5 D-07: warn when HIGH-risk s3dlio env vars are set.
+        self._check_storage_backend_env()
 
     def _run_fs_separation_probe(self) -> None:
         """Run the CAP-03 FS-separation probe and write the sidecar.

mlpstorage_py/cli/checkpointing_args.py

@@ -7,7 +7,10 @@
 
 import sys
 
-from mlpstorage_py.config import DEFAULT_HOSTS, EXEC_TYPE, LLM_MODELS, LLM_MODELS_CLOSED, EXIT_CODE
+from mlpstorage_py.config import (
+    DEFAULT_HOSTS, EXEC_TYPE, LLM_MODELS, LLM_MODELS_CLOSED, EXIT_CODE,
+    ENV_FALLBACK_CHECKPOINT_FOLDER,
+)
 from mlpstorage_py.cli.common_args import (
     HELP_MESSAGES,
     add_universal_arguments,
@@ -137,14 +140,27 @@ def _add_checkpointing_core_args(parser, command):
             ),
         )
 
-    # Checkpoint folder required for run only
+    # Checkpoint folder required for run only. The argparse-level required=True
+    # was replaced by post-parse enforcement (Phase 5 D-08 / D-09 / ENV-06) so
+    # the ENV_FALLBACK_CHECKPOINT_FOLDER default (sourced from
+    # MLPSTORAGE_CHECKPOINT_FOLDER env var) can satisfy the requirement. The
+    # loud-error gate lives in cli_parser._check_universal_required_present
+    # keyed off the _mlps_req_checkpoint_folder marker we set below. The
+    # --checkpoint-folder argument stays defined ONLY here (never in
+    # common_args.py) so no other benchmark inherits it.
     if command == "run":
         parser.add_argument(
             '--checkpoint-folder', '-cf',
             type=str,
-            required=True,
+            default=ENV_FALLBACK_CHECKPOINT_FOLDER,
             help=HELP_MESSAGES['checkpoint_folder']
         )
+        # Plumb the post-parse-gate marker locally, next to the arg definition,
+        # so the flag and its enforcement gate stay co-located (D-09). The
+        # symmetric req_checkpoint_folder kwarg on add_universal_arguments
+        # exists for defensive future callers; the checkpointing flow uses
+        # this local set_defaults.
+        parser.set_defaults(_mlps_req_checkpoint_folder=True)
 
     # num-checkpoints-read/write are available in all modes so closed submitters
     # can split write and read into two invocations (set =0 on one side) with a

mlpstorage_py/cli/common_args.py

@@ -12,7 +12,7 @@
 
 from mlpstorage_py.config import (
     CHECKPOINT_RANKS_STRINGS, MODELS, ACCELERATORS, ACCELERATORS_CLOSED, DEFAULT_HOSTS,
-    LLM_MODELS_STRINGS, MPI_CMDS, EXEC_TYPE, DEFAULT_RESULTS_DIR, DEFAULT_SYSTEMNAME,
+    LLM_MODELS_STRINGS, MPI_CMDS, EXEC_TYPE, ENV_FALLBACK_RESULTS_DIR, ENV_FALLBACK_SYSTEMNAME,
     VECTOR_DTYPES, DISTRIBUTIONS
 )
 
@@ -129,7 +129,10 @@ def add_arguments(self, actions):
     'reportgen': "Generate a report from the benchmark results.",
 
     # Checkpoint folder is used for training and checkpointing
-    'checkpoint_folder': "Location for checkpoint files for training or checkpointing workloads",
+    'checkpoint_folder': (
+        "Location for checkpoint files for training or checkpointing workloads. "
+        "Defaults to MLPSTORAGE_CHECKPOINT_FOLDER env var if set."
+    ),
 
     # Checkpointing help messages
     'checkpoint_run': "The checkpoint command executes checkpoint saves and restores for a given model.",
@@ -186,7 +189,7 @@ def add_arguments(self, actions):
     # System-under-test name (LAY-04 / D-10) — folder under results/ in canonical layout
     'systemname': (
         "System-under-test name (folder under results/). "
-        "Defaults to MLPERF_SYSTEMNAME env var if set."
+        "Defaults to MLPSTORAGE_SYSTEMNAME env var if set."
     ),
 
     # MPI help messages
@@ -219,22 +222,34 @@ def add_arguments(self, actions):
 }
 
 
-def add_universal_arguments(parser, req_results, req_systemname=False):
+def add_universal_arguments(parser, req_results, req_systemname=False, req_checkpoint_folder=False):
     """Add arguments common to all benchmarks and commands.
 
     Args:
         parser: Argparse parser to add arguments to.
         req_results: Whether --results-dir is required. The requirement is
             enforced POST-PARSE (see ``check_universal_arguments_present``)
-            so the ``MLPERF_RESULTS_DIR`` env-var-sourced default counts as
-            satisfying it. Using ``required=True`` here would short-circuit
+            so the ``MLPSTORAGE_RESULTS_DIR`` env-var-sourced default counts
+            as satisfying it. Using ``required=True`` here would short-circuit
             argparse before the env-var default applies — that's the CR-02
             bug we fixed.
         req_systemname: Whether --systemname is required. Defaults to False
             so pure utility call sites (lockfile, etc.) keep working without
             opting in. Emitting commands (datagen/run/configview/datasize/
             reportgen/history) MUST opt in per CONTEXT.md D-10 / LAY-04.
             Same post-parse enforcement model as ``req_results``.
+        req_checkpoint_folder: Whether --checkpoint-folder is required for
+            this invocation. Defaults to False. When True, plumbs the
+            ``_mlps_req_checkpoint_folder`` marker onto the argparse namespace
+            via ``parser.set_defaults`` so the post-parse gate
+            (``cli_parser._check_universal_required_present``) can enforce
+            non-emptiness with the D-02 loud-error line. NOTE: the
+            ``--checkpoint-folder`` argument itself is registered in
+            ``mlpstorage_py/cli/checkpointing_args.py`` — NOT here — so no
+            other benchmark accidentally inherits it (see Phase 5 D-09). This
+            kwarg exists for symmetry with ``req_results`` / ``req_systemname``
+            and for defensive future callers; the checkpointing flow plumbs
+            the marker locally in its own arg builder.
     """
     standard_args = parser.add_argument_group("Standard Arguments")
     # NOTE on the env-var fallback (CR-02): argparse's ``required=True``
@@ -245,11 +260,12 @@ def add_universal_arguments(parser, req_results, req_systemname=False):
     # emitting subcommand. We drop ``required=True`` entirely and instead
     # stash a marker on the namespace (via an argparse.SUPPRESS-defaulted
     # action) that ``check_universal_arguments_present`` consumes at the
-    # post-parse validation layer to enforce non-emptiness.
+    # post-parse validation layer to enforce non-emptiness. Env-var names:
+    # MLPSTORAGE_RESULTS_DIR / MLPSTORAGE_SYSTEMNAME (see config.py D-10).
     standard_args.add_argument(
         '--results-dir', '-rd',
         type=str,
-        default=DEFAULT_RESULTS_DIR,
+        default=ENV_FALLBACK_RESULTS_DIR,
         help=HELP_MESSAGES['results_dir']
     )
     if req_results:
@@ -260,19 +276,29 @@ def add_universal_arguments(parser, req_results, req_systemname=False):
         parser.set_defaults(_mlps_req_results=True)
 
     # --systemname: required on emitting commands (Rules.md §2.1.8 systemname
-    # subdir), optional elsewhere. Default = DEFAULT_SYSTEMNAME = "" unless
-    # MLPERF_SYSTEMNAME env var is set. Same env-var-fallback bug as above:
-    # we keep argparse's ``required=False`` and gate emptiness post-parse.
+    # subdir), optional elsewhere. Default sourced from ENV_FALLBACK_SYSTEMNAME
+    # = "" unless the MLPSTORAGE_SYSTEMNAME env var is set. Same env-var-fallback
+    # bug as above: we keep argparse's ``required=False`` and gate emptiness
+    # post-parse.
     standard_args.add_argument(
         '--systemname', '-sn',
         type=str,
         required=False,
-        default=DEFAULT_SYSTEMNAME,
+        default=ENV_FALLBACK_SYSTEMNAME,
         help=HELP_MESSAGES['systemname']
     )
     if req_systemname:
         parser.set_defaults(_mlps_req_systemname=True)
 
+    # --checkpoint-folder marker: per D-09, the `--checkpoint-folder` argument
+    # is registered in checkpointing_args.py (NOT here). This branch only plumbs
+    # the ``_mlps_req_checkpoint_folder`` namespace marker for callers that
+    # prefer symmetric opt-in via ``add_universal_arguments``. The checkpointing
+    # flow instead plumbs this marker locally next to its arg definition to keep
+    # the flag and its enforcement gate co-located.
+    if req_checkpoint_folder:
+        parser.set_defaults(_mlps_req_checkpoint_folder=True)
+
     standard_args.add_argument(
         '--config-file', '-c',
         type=str,

mlpstorage_py/cli/training_args.py

@@ -6,11 +6,12 @@
 """
 
 import argparse
+import os
 import sys
 
 from mlpstorage_py.config import (
     MODELS, MODELS_CLOSED, MODELS_OPEN, ACCELERATORS, ACCELERATORS_CLOSED,
-    DEFAULT_HOSTS, EXEC_TYPE, EXIT_CODE
+    DEFAULT_HOSTS, EXEC_TYPE, EXIT_CODE, ENV_FALLBACK_DATA_DIR
 )
 
 from mlpstorage_py.cli.common_args import (
@@ -171,9 +172,11 @@ def _add_training_core_args(parser, command, accel_choices):
     parser.add_argument(
         '--data-dir', '-dd',
         type=str,
+        default=ENV_FALLBACK_DATA_DIR,
         help=(
             "Dataset location. For file storage, this is a filesystem path. "
-            "For object storage, this is an object key prefix or full object URI."
+            "For object storage, this is an object key prefix or full object URI. "
+            "Defaults to MLPSTORAGE_DATA_DIR env var if set."
         )
     )
 
@@ -293,14 +296,21 @@ def validate_training_arguments(args):
 
     # Object-mode --data-dir enforcement runs here (post-YAML) so --config-file
     # can populate data_dir for object workflows. File-mode enforcement happens
-    # earlier in cli_parser.parse_arguments via parser.error().
+    # earlier in cli_parser.parse_arguments via parser.error(). Message text is
+    # the D-02 verbatim template (Phase 5 D-02 / D-07) — do NOT paraphrase; it
+    # is pinned by loud-error tests.
     if command in ('datagen', 'run') and protocol == 'object' and not getattr(args, 'data_dir', None):
         print(
-            f"ERROR: --data-dir is required for training {command} with object storage.\n"
-            "  Specify --data-dir <key-prefix-or-URI> on the command line, or set\n"
-            "  'data_dir:' in the file passed via --config-file.",
+            f"error: --data-dir/-dd is required: pass it on the command line or set MLPSTORAGE_DATA_DIR",
             file=sys.stderr,
         )
+        # D-05 migration hint: fires only when the user set the legacy env var
+        # but not the new one — actionable at exactly the moment of failure.
+        if os.environ.get('MLPERF_DATA_DIR') and not os.environ.get('MLPSTORAGE_DATA_DIR'):
+            print(
+                f"hint: MLPERF_DATA_DIR is set but is no longer read; rename it to MLPSTORAGE_DATA_DIR",
+                file=sys.stderr,
+            )
         sys.exit(EXIT_CODE.INVALID_ARGUMENTS)
 
     if getattr(args, 'o_direct', False) and protocol != 'file':