From 0889d5cb275e40846b178f10ae59df7d9a0eb784 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:43:36 -0700 Subject: [PATCH 01/45] refactor(05-01): consolidate MLPSTORAGE_* env-var names, add ENV_FALLBACK_* resolvers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Introduce 5 MLPSTORAGE_*_ENVVAR string constants as single source of truth (D-10). - Add _LEGACY_ENVVAR_MAP for migration-hint lookup (excludes CHECKPOINT_FOLDER per D-08). - Rename _resolve_default_* → _resolve_env_fallback_*, DEFAULT_* → ENV_FALLBACK_*. - Add resolvers/constants for DATA_DIR and CHECKPOINT_FOLDER. - Delete tempdir results-dir fallback and drop unused `import tempfile` (D-13, D-14). - All resolvers return "" when env var unset (SC-3): never None, never a tempdir path. --- mlpstorage_py/config.py | 97 +++++++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 32 deletions(-) diff --git a/mlpstorage_py/config.py b/mlpstorage_py/config.py index b6e8f873..694b4104 100755 --- a/mlpstorage_py/config.py +++ b/mlpstorage_py/config.py @@ -2,7 +2,6 @@ import enum import os import pathlib -import tempfile def check_env(setting, default_value=None): @@ -144,40 +143,74 @@ def get_datetime_string(): MAX_NUM_FILES_TRAIN = 128*1024 -def _resolve_default_results_dir() -> str: - """Resolve DEFAULT_RESULTS_DIR from MLPERF_RESULTS_DIR or tempdir. +# ----------------------------------------------------------------------------- +# MLPSTORAGE_* env-var-name string constants — SINGLE SOURCE OF TRUTH (D-10). +# +# Every mlpstorage-owned environment-variable NAME lives here as a string +# constant. Downstream modules (rules/utils.py, cli_parser.py, etc.) MUST +# import these names from this module rather than redefining them locally. +# Import direction is one-way (D-11): config.py MUST NOT import from +# mlpstorage_py.rules.*; doing so would create a cycle at interpreter +# startup. tests/unit/test_no_import_cycles.py locks this invariant. +# ----------------------------------------------------------------------------- +MLPSTORAGE_ORGNAME_ENVVAR = "MLPSTORAGE_ORGNAME" +MLPSTORAGE_SYSTEMNAME_ENVVAR = "MLPSTORAGE_SYSTEMNAME" +MLPSTORAGE_RESULTS_DIR_ENVVAR = "MLPSTORAGE_RESULTS_DIR" +MLPSTORAGE_DATA_DIR_ENVVAR = "MLPSTORAGE_DATA_DIR" +MLPSTORAGE_CHECKPOINT_FOLDER_ENVVAR = "MLPSTORAGE_CHECKPOINT_FOLDER" + +# _LEGACY_ENVVAR_MAP: new MLPSTORAGE_* env-var-name -> legacy MLPERF_* +# predecessor. Consumed by the parse-time migration-hint emitter (Plan 05-02 +# D-04/D-05). MLPSTORAGE_CHECKPOINT_FOLDER_ENVVAR is intentionally absent — +# per D-08 no MLPERF_CHECKPOINT_FOLDER ever existed, so the migration hint +# never fires for that pair. +_LEGACY_ENVVAR_MAP = { + MLPSTORAGE_SYSTEMNAME_ENVVAR: "MLPERF_SYSTEMNAME", + MLPSTORAGE_RESULTS_DIR_ENVVAR: "MLPERF_RESULTS_DIR", + MLPSTORAGE_DATA_DIR_ENVVAR: "MLPERF_DATA_DIR", + MLPSTORAGE_ORGNAME_ENVVAR: "MLPERF_ORGNAME", +} - Extracted so tests can exercise the env-driven branch without - reloading this module — reload re-creates the PARAM_VALIDATION enum - class, which then fails `in`-checks against any pre-imported copy - held by other modules (notably the rules verifier). - """ - return os.environ.get( - "MLPERF_RESULTS_DIR", - os.path.join(tempfile.gettempdir(), "mlperf_storage_results"), - ) - - -DEFAULT_RESULTS_DIR = _resolve_default_results_dir() - -# DEFAULT_SYSTEMNAME mirrors DEFAULT_RESULTS_DIR (LAY-04): honor the -# MLPERF_SYSTEMNAME env var if set, fall back to an empty string. The -# universal-args layer (add_universal_arguments) decides required-vs-optional -# per subcommand; an empty default plus required=True on emitting commands -# makes "no --systemname and no env var" fail at parse time rather than -# silently producing `///results//...` (T-1-02). -def _resolve_default_systemname() -> str: - """Resolve DEFAULT_SYSTEMNAME from MLPERF_SYSTEMNAME. - - Extracted so tests can exercise the env-driven branch without - reloading this module — reload re-creates the PARAM_VALIDATION enum - class, which then fails `in`-checks against any pre-imported copy - held by other modules (notably the rules verifier). - """ - return os.environ.get("MLPERF_SYSTEMNAME", "") + +# ----------------------------------------------------------------------------- +# ENV_FALLBACK_* module-level constants — resolved-at-import-time env-var +# reads for the four universal path/name arguments. Every resolver returns +# the empty string when its env var is unset (SC-3): never None, never a +# tempdir path. The universal-args layer (add_universal_arguments) decides +# required-vs-optional per subcommand; an empty fallback plus the parse-time +# loud-error gate (Plan 05-02) makes "no --flag and no env var" fail at +# parse time rather than silently producing malformed output paths (T-1-02). +# +# Resolvers are extracted from the module-level assignments so tests can +# exercise the env-driven branch without reloading this module — reload +# re-creates the PARAM_VALIDATION enum class, which then fails `in`-checks +# against any pre-imported copy held by other modules (notably the rules +# verifier). See LAY-04 for the same shape mirrored across all four. +# ----------------------------------------------------------------------------- +def _resolve_env_fallback_results_dir() -> str: + """Return MLPSTORAGE_RESULTS_DIR or empty string (never a tempdir path).""" + return os.environ.get(MLPSTORAGE_RESULTS_DIR_ENVVAR, "") + + +def _resolve_env_fallback_systemname() -> str: + """Return MLPSTORAGE_SYSTEMNAME or empty string.""" + return os.environ.get(MLPSTORAGE_SYSTEMNAME_ENVVAR, "") + + +def _resolve_env_fallback_data_dir() -> str: + """Return MLPSTORAGE_DATA_DIR or empty string.""" + return os.environ.get(MLPSTORAGE_DATA_DIR_ENVVAR, "") + + +def _resolve_env_fallback_checkpoint_folder() -> str: + """Return MLPSTORAGE_CHECKPOINT_FOLDER or empty string.""" + return os.environ.get(MLPSTORAGE_CHECKPOINT_FOLDER_ENVVAR, "") -DEFAULT_SYSTEMNAME = _resolve_default_systemname() +ENV_FALLBACK_RESULTS_DIR = _resolve_env_fallback_results_dir() +ENV_FALLBACK_SYSTEMNAME = _resolve_env_fallback_systemname() +ENV_FALLBACK_DATA_DIR = _resolve_env_fallback_data_dir() +ENV_FALLBACK_CHECKPOINT_FOLDER = _resolve_env_fallback_checkpoint_folder() import enum From b9a1bab502674b1b4d4bf67eaacfed246c5e1fc1 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:44:56 -0700 Subject: [PATCH 02/45] refactor(05-01): drop tempdir-fallback warn block; rename downstream DEFAULT_* refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete the 10-line dead-code warn block at main.py:247–256 (D-13); the parse-time loud-error gate in Plan 05-02 replaces its job. - Drop DEFAULT_RESULTS_DIR from main.py's config import; rewrite the stray reference at the reports-subcommand dispatch to getattr(args, 'results_dir', ""). - [Rule 3] Rename common_args.py's DEFAULT_RESULTS_DIR / DEFAULT_SYSTEMNAME references to ENV_FALLBACK_RESULTS_DIR / ENV_FALLBACK_SYSTEMNAME; without this, `import mlpstorage_py.main` (a Task-2 acceptance criterion) raises ImportError against the renamed constants from Task 1. --- mlpstorage_py/cli/common_args.py | 10 +++++----- mlpstorage_py/main.py | 15 ++------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/mlpstorage_py/cli/common_args.py b/mlpstorage_py/cli/common_args.py index caf6572a..21086abc 100755 --- a/mlpstorage_py/cli/common_args.py +++ b/mlpstorage_py/cli/common_args.py @@ -12,7 +12,7 @@ from mlpstorage_py.config import ( CHECKPOINT_RANKS_STRINGS, MODELS, ACCELERATORS, ACCELERATORS_CLOSED, DEFAULT_HOSTS, - LLM_MODELS_STRINGS, MPI_CMDS, EXEC_TYPE, DEFAULT_RESULTS_DIR, DEFAULT_SYSTEMNAME, + LLM_MODELS_STRINGS, MPI_CMDS, EXEC_TYPE, ENV_FALLBACK_RESULTS_DIR, ENV_FALLBACK_SYSTEMNAME, VECTOR_DTYPES, DISTRIBUTIONS ) @@ -249,7 +249,7 @@ def add_universal_arguments(parser, req_results, req_systemname=False): standard_args.add_argument( '--results-dir', '-rd', type=str, - default=DEFAULT_RESULTS_DIR, + default=ENV_FALLBACK_RESULTS_DIR, help=HELP_MESSAGES['results_dir'] ) if req_results: @@ -260,14 +260,14 @@ def add_universal_arguments(parser, req_results, req_systemname=False): parser.set_defaults(_mlps_req_results=True) # --systemname: required on emitting commands (Rules.md §2.1.8 systemname - # subdir), optional elsewhere. Default = DEFAULT_SYSTEMNAME = "" unless - # MLPERF_SYSTEMNAME env var is set. Same env-var-fallback bug as above: + # subdir), optional elsewhere. Default = ENV_FALLBACK_SYSTEMNAME = "" unless + # MLPSTORAGE_SYSTEMNAME env var is set. Same env-var-fallback bug as above: # we keep argparse's ``required=False`` and gate emptiness post-parse. standard_args.add_argument( '--systemname', '-sn', type=str, required=False, - default=DEFAULT_SYSTEMNAME, + default=ENV_FALLBACK_SYSTEMNAME, help=HELP_MESSAGES['systemname'] ) if req_systemname: diff --git a/mlpstorage_py/main.py b/mlpstorage_py/main.py index 7bf04ebc..8177cdfd 100755 --- a/mlpstorage_py/main.py +++ b/mlpstorage_py/main.py @@ -15,7 +15,7 @@ import traceback from mlpstorage_py.cli_parser import parse_arguments, validate_args, update_args -from mlpstorage_py.config import HISTFILE, DATETIME_STR, EXIT_CODE, DEFAULT_RESULTS_DIR, get_datetime_string, HYDRA_OUTPUT_SUBDIR +from mlpstorage_py.config import HISTFILE, DATETIME_STR, EXIT_CODE, get_datetime_string, HYDRA_OUTPUT_SUBDIR from mlpstorage_py.debug import debugger_hook, MLPS_DEBUG from mlpstorage_py.history import HistoryTracker from mlpstorage_py.mlps_logging import setup_logging, apply_logging_options @@ -244,17 +244,6 @@ def run_benchmark(args, run_datetime): benchmark = benchmark_class(args, run_datetime=run_datetime, logger=logger) - # Warn if the user is relying on the temp-dir default for results. - # Results stored in /tmp (or equivalent) are wiped on reboot. - _results_dir = getattr(args, 'results_dir', DEFAULT_RESULTS_DIR) - if _results_dir == DEFAULT_RESULTS_DIR and not os.environ.get('MLPERF_RESULTS_DIR'): - logger.warning( - f"Results directory not specified. Writing results to the system temp directory: " - f"{DEFAULT_RESULTS_DIR}. These results will NOT persist across a reboot. " - f"Use --results-dir or set the MLPERF_RESULTS_DIR environment variable " - f"to save results permanently." - ) - ret_code = EXIT_CODE.SUCCESS try: @@ -435,7 +424,7 @@ def _main_impl(): # for the reports subcommand. Keeping the import here lets validate / # rules-coverage / version run on a base install. from mlpstorage_py.report_generator import ReportGenerator - results_dir = args.results_dir if hasattr(args, 'results_dir') else DEFAULT_RESULTS_DIR + results_dir = getattr(args, 'results_dir', "") report_generator = ReportGenerator(results_dir, args, logger=logger) return report_generator.generate_reports() From f2b9d761b37103ba11a0006a7b1a564bada8ac8b Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:45:19 -0700 Subject: [PATCH 03/45] refactor(05-01): import MLPSTORAGE_*_ENVVAR names from config, not local redefinition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extend the `from mlpstorage_py.config import ...` block to pull MLPSTORAGE_ORGNAME_ENVVAR and MLPSTORAGE_SYSTEMNAME_ENVVAR from config.py. - Delete the two local string-constant assignments (previously rules/utils.py:21-22). - Preserves the one-way dependency invariant: config → rules.utils (D-11). Downstream callers reading `from mlpstorage_py.rules.utils import MLPSTORAGE_*_ENVVAR` still work because the config import re-exposes the names in this module's namespace. --- mlpstorage_py/rules/utils.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/mlpstorage_py/rules/utils.py b/mlpstorage_py/rules/utils.py index 3af09e50..a3877dac 100755 --- a/mlpstorage_py/rules/utils.py +++ b/mlpstorage_py/rules/utils.py @@ -10,16 +10,25 @@ import sys from typing import Tuple, List, Optional -from mlpstorage_py.config import BENCHMARK_TYPES, DATETIME_STR +from mlpstorage_py.config import ( + BENCHMARK_TYPES, + DATETIME_STR, + MLPSTORAGE_ORGNAME_ENVVAR, + MLPSTORAGE_SYSTEMNAME_ENVVAR, +) from mlpstorage_py.errors import ConfigurationError, ErrorCode -# Env-var names used by the CLI dispatch layer to source orgname/systemname. -# generate_output_location itself does NOT read these; values are threaded in -# via benchmark.args (populated upstream by main._main_impl()'s sentinel- -# resolution gate). The names are exported here as a single source of truth -# for the env-var spelling. -MLPSTORAGE_ORGNAME_ENVVAR = "MLPSTORAGE_ORGNAME" -MLPSTORAGE_SYSTEMNAME_ENVVAR = "MLPSTORAGE_SYSTEMNAME" +# Env-var names used by the CLI dispatch layer to source orgname/systemname +# are imported from mlpstorage_py.config (D-10): config.py is the single +# source of truth for the env-var spelling. Import direction is one-way +# (D-11) — config.py MUST NOT import from mlpstorage_py.rules. Downstream +# callers that historically read `from mlpstorage_py.rules.utils import +# MLPSTORAGE_*_ENVVAR` continue to work because these names are re-exported +# by the config import above. +# +# generate_output_location itself does NOT read these; values are threaded +# in via benchmark.args (populated upstream by main._main_impl()'s +# sentinel-resolution gate). # Each path segment appended to results_dir by generate_output_location must # match this — POSIX-safe alphanumeric plus '.', '_', '-' — and must not be From 7c3b4ed74781af764d7ccbc7aeebd7768e22be9b Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:48:18 -0700 Subject: [PATCH 04/45] refactor(05-04): rename run_summary Environment MLPERF_* rows to MLPSTORAGE_* - Rename MLPERF_RESULTS_DIR and MLPERF_SYSTEMNAME rows to their MLPSTORAGE_* counterparts in the run summary Environment section. - Add three sibling rows for MLPSTORAGE_ORGNAME, MLPSTORAGE_DATA_DIR, and MLPSTORAGE_CHECKPOINT_FOLDER so the diagnostic surface reflects the full set of owned env vars. - Order: RESULTS_DIR, SYSTEMNAME, ORGNAME, DATA_DIR, CHECKPOINT_FOLDER, then MPI_RUN_BIN, MPI_EXEC_BIN (unchanged), then existing KVCACHE_* block. - MPI_RUN_BIN, MPI_EXEC_BIN, and KVCACHE_SELECTED_WORKLOADS rows unchanged. --- mlpstorage_py/run_summary.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mlpstorage_py/run_summary.py b/mlpstorage_py/run_summary.py index 9701277a..6706b46c 100644 --- a/mlpstorage_py/run_summary.py +++ b/mlpstorage_py/run_summary.py @@ -530,10 +530,13 @@ def print_run_summary(args) -> None: # Always-visible environment section. lines.append("") lines.append("--- Environment ---") - lines.append(_row("MLPERF_RESULTS_DIR:", os.environ.get('MLPERF_RESULTS_DIR', '[not set]'))) - lines.append(_row("MLPERF_SYSTEMNAME:", os.environ.get('MLPERF_SYSTEMNAME', '[not set]'))) - lines.append(_row("MPI_RUN_BIN:", os.environ.get('MPI_RUN_BIN', '[not set]'))) - lines.append(_row("MPI_EXEC_BIN:", os.environ.get('MPI_EXEC_BIN', '[not set]'))) + lines.append(_row("MLPSTORAGE_RESULTS_DIR:", os.environ.get('MLPSTORAGE_RESULTS_DIR', '[not set]'))) + lines.append(_row("MLPSTORAGE_SYSTEMNAME:", os.environ.get('MLPSTORAGE_SYSTEMNAME', '[not set]'))) + lines.append(_row("MLPSTORAGE_ORGNAME:", os.environ.get('MLPSTORAGE_ORGNAME', '[not set]'))) + lines.append(_row("MLPSTORAGE_DATA_DIR:", os.environ.get('MLPSTORAGE_DATA_DIR', '[not set]'))) + lines.append(_row("MLPSTORAGE_CHECKPOINT_FOLDER:", os.environ.get('MLPSTORAGE_CHECKPOINT_FOLDER', '[not set]'))) + lines.append(_row("MPI_RUN_BIN:", os.environ.get('MPI_RUN_BIN', '[not set]'))) + lines.append(_row("MPI_EXEC_BIN:", os.environ.get('MPI_EXEC_BIN', '[not set]'))) # KVCACHE_SELECTED_WORKLOADS is read by kv-cache-wrapper.sh and filters the # workload set. Surface it for kvcache so reviewers know whether the full # suite or a subset ran. From 3113ef0093174fb4e1f0f95747c80c42677fed8d Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:48:55 -0700 Subject: [PATCH 05/45] feat(05-03): add req_checkpoint_folder kwarg + MLPSTORAGE_ help text - add_universal_arguments grows a third opt-in kwarg req_checkpoint_folder (default False) that plumbs the _mlps_req_checkpoint_folder marker via parser.set_defaults, mirroring the req_results / req_systemname pattern (D-09). The --checkpoint-folder arg itself stays in checkpointing_args.py so no other benchmark inherits it. - HELP_MESSAGES['systemname'] now references MLPSTORAGE_SYSTEMNAME (not MLPERF_SYSTEMNAME). Marker-plumbing NOTE block updated to name MLPSTORAGE_RESULTS_DIR / MLPSTORAGE_SYSTEMNAME as the env-var-fallback sources. - Docstring documents the new kwarg and points readers at D-09. --- mlpstorage_py/cli/common_args.py | 39 +++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/mlpstorage_py/cli/common_args.py b/mlpstorage_py/cli/common_args.py index 21086abc..17ba56e5 100755 --- a/mlpstorage_py/cli/common_args.py +++ b/mlpstorage_py/cli/common_args.py @@ -186,7 +186,7 @@ def add_arguments(self, actions): # System-under-test name (LAY-04 / D-10) — folder under results/ in canonical layout 'systemname': ( "System-under-test name (folder under results/). " - "Defaults to MLPERF_SYSTEMNAME env var if set." + "Defaults to MLPSTORAGE_SYSTEMNAME env var if set." ), # MPI help messages @@ -219,15 +219,15 @@ def add_arguments(self, actions): } -def add_universal_arguments(parser, req_results, req_systemname=False): +def add_universal_arguments(parser, req_results, req_systemname=False, req_checkpoint_folder=False): """Add arguments common to all benchmarks and commands. Args: parser: Argparse parser to add arguments to. req_results: Whether --results-dir is required. The requirement is enforced POST-PARSE (see ``check_universal_arguments_present``) - so the ``MLPERF_RESULTS_DIR`` env-var-sourced default counts as - satisfying it. Using ``required=True`` here would short-circuit + so the ``MLPSTORAGE_RESULTS_DIR`` env-var-sourced default counts + as satisfying it. Using ``required=True`` here would short-circuit argparse before the env-var default applies — that's the CR-02 bug we fixed. req_systemname: Whether --systemname is required. Defaults to False @@ -235,6 +235,18 @@ def add_universal_arguments(parser, req_results, req_systemname=False): opting in. Emitting commands (datagen/run/configview/datasize/ reportgen/history) MUST opt in per CONTEXT.md D-10 / LAY-04. Same post-parse enforcement model as ``req_results``. + req_checkpoint_folder: Whether --checkpoint-folder is required for + this invocation. Defaults to False. When True, plumbs the + ``_mlps_req_checkpoint_folder`` marker onto the argparse namespace + via ``parser.set_defaults`` so the post-parse gate + (``cli_parser._check_universal_required_present``) can enforce + non-emptiness with the D-02 loud-error line. NOTE: the + ``--checkpoint-folder`` argument itself is registered in + ``mlpstorage_py/cli/checkpointing_args.py`` — NOT here — so no + other benchmark accidentally inherits it (see Phase 5 D-09). This + kwarg exists for symmetry with ``req_results`` / ``req_systemname`` + and for defensive future callers; the checkpointing flow plumbs + the marker locally in its own arg builder. """ standard_args = parser.add_argument_group("Standard Arguments") # NOTE on the env-var fallback (CR-02): argparse's ``required=True`` @@ -245,7 +257,8 @@ def add_universal_arguments(parser, req_results, req_systemname=False): # emitting subcommand. We drop ``required=True`` entirely and instead # stash a marker on the namespace (via an argparse.SUPPRESS-defaulted # action) that ``check_universal_arguments_present`` consumes at the - # post-parse validation layer to enforce non-emptiness. + # post-parse validation layer to enforce non-emptiness. Env-var names: + # MLPSTORAGE_RESULTS_DIR / MLPSTORAGE_SYSTEMNAME (see config.py D-10). standard_args.add_argument( '--results-dir', '-rd', type=str, @@ -260,9 +273,10 @@ def add_universal_arguments(parser, req_results, req_systemname=False): parser.set_defaults(_mlps_req_results=True) # --systemname: required on emitting commands (Rules.md §2.1.8 systemname - # subdir), optional elsewhere. Default = ENV_FALLBACK_SYSTEMNAME = "" unless - # MLPSTORAGE_SYSTEMNAME env var is set. Same env-var-fallback bug as above: - # we keep argparse's ``required=False`` and gate emptiness post-parse. + # subdir), optional elsewhere. Default sourced from ENV_FALLBACK_SYSTEMNAME + # = "" unless the MLPSTORAGE_SYSTEMNAME env var is set. Same env-var-fallback + # bug as above: we keep argparse's ``required=False`` and gate emptiness + # post-parse. standard_args.add_argument( '--systemname', '-sn', type=str, @@ -273,6 +287,15 @@ def add_universal_arguments(parser, req_results, req_systemname=False): if req_systemname: parser.set_defaults(_mlps_req_systemname=True) + # --checkpoint-folder marker: per D-09, the `--checkpoint-folder` argument + # is registered in checkpointing_args.py (NOT here). This branch only plumbs + # the ``_mlps_req_checkpoint_folder`` namespace marker for callers that + # prefer symmetric opt-in via ``add_universal_arguments``. The checkpointing + # flow instead plumbs this marker locally next to its arg definition to keep + # the flag and its enforcement gate co-located. + if req_checkpoint_folder: + parser.set_defaults(_mlps_req_checkpoint_folder=True) + standard_args.add_argument( '--config-file', '-c', type=str, From 7f013c6822789ca690468c0d24c451deb05b34be Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:49:34 -0700 Subject: [PATCH 06/45] feat(05-03): rewrite training --data-dir gate to D-02 verbatim template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - --data-dir now defaults to ENV_FALLBACK_DATA_DIR (from MLPSTORAGE_DATA_DIR env var if set) so users can seed data location via env alone (ENV-03). - Post-YAML gate in validate_training_arguments now emits the D-02 verbatim loud-error line: "error: --data-dir/-dd is required: pass it on the command line or set MLPSTORAGE_DATA_DIR" Gate location (post-YAML, object-mode-only guard) is byte-preserved per D-07 so --config-file YAML data_dir: still counts. - D-05 migration hint fires immediately below the error line when the user has MLPERF_DATA_DIR set but MLPSTORAGE_DATA_DIR unset — actionable at the exact moment of failure. Hint uses the verbatim D-05 template. - Adds `import os` for the environ lookup. --- mlpstorage_py/cli/training_args.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mlpstorage_py/cli/training_args.py b/mlpstorage_py/cli/training_args.py index 55a6ba7c..53a27bd6 100755 --- a/mlpstorage_py/cli/training_args.py +++ b/mlpstorage_py/cli/training_args.py @@ -6,11 +6,12 @@ """ import argparse +import os import sys from mlpstorage_py.config import ( MODELS, MODELS_CLOSED, MODELS_OPEN, ACCELERATORS, ACCELERATORS_CLOSED, - DEFAULT_HOSTS, EXEC_TYPE, EXIT_CODE + DEFAULT_HOSTS, EXEC_TYPE, EXIT_CODE, ENV_FALLBACK_DATA_DIR ) from mlpstorage_py.cli.common_args import ( @@ -171,9 +172,11 @@ def _add_training_core_args(parser, command, accel_choices): parser.add_argument( '--data-dir', '-dd', type=str, + default=ENV_FALLBACK_DATA_DIR, help=( "Dataset location. For file storage, this is a filesystem path. " - "For object storage, this is an object key prefix or full object URI." + "For object storage, this is an object key prefix or full object URI. " + "Defaults to MLPSTORAGE_DATA_DIR env var if set." ) ) @@ -293,14 +296,21 @@ def validate_training_arguments(args): # Object-mode --data-dir enforcement runs here (post-YAML) so --config-file # can populate data_dir for object workflows. File-mode enforcement happens - # earlier in cli_parser.parse_arguments via parser.error(). + # earlier in cli_parser.parse_arguments via parser.error(). Message text is + # the D-02 verbatim template (Phase 5 D-02 / D-07) — do NOT paraphrase; it + # is pinned by loud-error tests. if command in ('datagen', 'run') and protocol == 'object' and not getattr(args, 'data_dir', None): print( - f"ERROR: --data-dir is required for training {command} with object storage.\n" - " Specify --data-dir on the command line, or set\n" - " 'data_dir:' in the file passed via --config-file.", + f"error: --data-dir/-dd is required: pass it on the command line or set MLPSTORAGE_DATA_DIR", file=sys.stderr, ) + # D-05 migration hint: fires only when the user set the legacy env var + # but not the new one — actionable at exactly the moment of failure. + if os.environ.get('MLPERF_DATA_DIR') and not os.environ.get('MLPSTORAGE_DATA_DIR'): + print( + f"hint: MLPERF_DATA_DIR is set but is no longer read; rename it to MLPSTORAGE_DATA_DIR", + file=sys.stderr, + ) sys.exit(EXIT_CODE.INVALID_ARGUMENTS) if getattr(args, 'o_direct', False) and protocol != 'file': From bcfa00757aede3a2dd003bcbd2633b1bf7a9715f Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:50:18 -0700 Subject: [PATCH 07/45] feat(05-02): rewrite _check_universal_required_present per D-01/D-02/D-04/D-05 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add _UNIVERSAL_REQUIRED_SPECS table: three rows for --results-dir/-rd, --systemname/-sn, --checkpoint-folder/-cf (D-09). Training's --data-dir is intentionally absent — its gate stays post-YAML in training_args.py (D-07) so --config-file can supply data_dir. - Emit ONE 'error:' line per missing required universal using the D-02 verbatim template: '--{long}/-{short} is required: pass it on the command line or set MLPSTORAGE_{NAME}'. Aggregate every missing flag before exiting once with EXIT_CODE.INVALID_ARGUMENTS (D-01, D-03, T-05-06). No comma-join, no first-error-then-exit. - Emit an adjacent 'hint:' line immediately below its associated 'error:' when the legacy MLPERF_* env var is set AND the new MLPSTORAGE_* is not, using the D-05 verbatim template: 'hint: MLPERF_{NAME} is set but is no longer read; rename it to MLPSTORAGE_{NAME}'. Lookup goes through _LEGACY_ENVVAR_MAP imported from config — no bare MLPERF_* string literals in cli_parser.py. - Since _LEGACY_ENVVAR_MAP has no key for MLPSTORAGE_CHECKPOINT_FOLDER (D-08 — no legacy MLPERF_CHECKPOINT_FOLDER ever existed), the hint never fires for checkpoint-folder; the exclusion falls out of the map lookup with no special-case code. - Rewrite the docstring to reflect the three-universal scope, per-flag emission model, adjacent-hint contract, and D-01/D-02/D-04/D-05/D-09 references. Drop obsolete mentions of MLPERF_RESULTS_DIR / MLPERF_SYSTEMNAME as arg-populating defaults. Behavioral checks (all three markers set, all values empty, env clean): * 3 'error:' lines, 0 'hint:' lines, exit code 2, verbatim templates. With MLPERF_SYSTEMNAME and MLPERF_RESULTS_DIR set, MLPSTORAGE_* unset: * 3 'error:' lines with hints IMMEDIATELY below the results/systemname errors, and none below checkpoint-folder. TEST-13 (Plan 05-06) will pin these exact strings; any future paraphrase fails the phase-wide grep pinned in the acceptance-criteria block of 05-02-PLAN.md. --- mlpstorage_py/cli_parser.py | 142 ++++++++++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 32 deletions(-) diff --git a/mlpstorage_py/cli_parser.py b/mlpstorage_py/cli_parser.py index f872ff41..15fee57a 100755 --- a/mlpstorage_py/cli_parser.py +++ b/mlpstorage_py/cli_parser.py @@ -6,12 +6,21 @@ """ import argparse +import os import re import shlex import sys from mlpstorage_py import VERSION -from mlpstorage_py.config import LLM_MODELS, VECTORDB_DEFAULT_RUNTIME, EXIT_CODE +from mlpstorage_py.config import ( + LLM_MODELS, + VECTORDB_DEFAULT_RUNTIME, + EXIT_CODE, + MLPSTORAGE_RESULTS_DIR_ENVVAR, + MLPSTORAGE_SYSTEMNAME_ENVVAR, + MLPSTORAGE_CHECKPOINT_FOLDER_ENVVAR, + _LEGACY_ENVVAR_MAP, +) # Import modular argument builders from cli package from mlpstorage_py.cli import ( @@ -37,6 +46,36 @@ prog_descriptions = PROGRAM_DESCRIPTIONS +# ----------------------------------------------------------------------------- +# _UNIVERSAL_REQUIRED_SPECS — declarative table for the parse-time +# required-universal gate (Plan 05-02 D-09). Each row describes one universal +# CLI flag that a subcommand may opt into as "required-with-env-fallback": +# +# (marker_attr_name, arg_attr_name, envvar_name_constant, +# long_flag, short_flag, name_for_template) +# +# - marker_attr_name: the ``args._mlps_req_*`` attribute set by +# ``add_universal_arguments(..., req_=True)`` on the opt-in subcommand. +# - arg_attr_name: the resolved argparse dest whose value we check for +# truthiness (env-var-sourced defaults populate this if the CLI flag was +# omitted). +# - envvar_name_constant: the ``MLPSTORAGE_*`` env-var-name string constant +# (single source of truth in ``mlpstorage_py.config``); also the lookup +# key into ``_LEGACY_ENVVAR_MAP`` for the D-05 migration hint. +# - long_flag / short_flag / name_for_template: substituted verbatim into +# the D-02 error template and D-05 hint template. +# +# Declaration order pins emission order (results, systemname, +# checkpoint-folder). Training's ``--data-dir`` is intentionally absent — +# per D-07 it gates in ``training_args.py`` after YAML merge. +# ----------------------------------------------------------------------------- +_UNIVERSAL_REQUIRED_SPECS = ( + ("_mlps_req_results", "results_dir", MLPSTORAGE_RESULTS_DIR_ENVVAR, "--results-dir", "-rd", "RESULTS_DIR"), + ("_mlps_req_systemname", "systemname", MLPSTORAGE_SYSTEMNAME_ENVVAR, "--systemname", "-sn", "SYSTEMNAME"), + ("_mlps_req_checkpoint_folder", "checkpoint_folder", MLPSTORAGE_CHECKPOINT_FOLDER_ENVVAR, "--checkpoint-folder", "-cf", "CHECKPOINT_FOLDER"), +) + + def _apply_formatter(parser): """Recursively set MLPStorageHelpFormatter on every parser in the subparser tree.""" parser.formatter_class = MLPStorageHelpFormatter @@ -285,38 +324,77 @@ def validate_args(args): def _check_universal_required_present(args): - """Enforce post-parse non-empty checks for --results-dir / --systemname. - - Subcommands opt in to "required" via ``add_universal_arguments(..., req_results=True)`` - and / or ``req_systemname=True``. That call seeds - ``args._mlps_req_results`` / ``args._mlps_req_systemname`` via - ``parser.set_defaults``. Here we just look at the resolved attribute - values; if the corresponding flag was required but the value is empty - (or None), exit with the same argparse-style error that argparse would - have emitted if ``required=True`` had been used directly. - - The env-var-sourced defaults (``MLPERF_RESULTS_DIR`` / - ``MLPERF_SYSTEMNAME``) populate ``args.results_dir`` / - ``args.systemname`` for free if argparse used the default; this check - is precisely what makes the env-var fallback live again on emitting - subcommands. + """Enforce post-parse non-empty checks for the required universal flags. + + Covers three universals per Plan 05-02 / D-09: + * ``--results-dir / -rd`` (marker: ``_mlps_req_results``) + * ``--systemname / -sn`` (marker: ``_mlps_req_systemname``) + * ``--checkpoint-folder / -cf`` (marker: ``_mlps_req_checkpoint_folder``) + + Training's ``--data-dir`` does NOT flow through this gate — it is + checked in ``training_args.py`` AFTER YAML config merge (D-07), because + a ``--config-file`` may legitimately supply ``data_dir``. + + Subcommands opt in to "required" via + ``add_universal_arguments(..., req_results=True, ...)`` and equivalents, + which set ``args._mlps_req_*`` markers via ``parser.set_defaults``. The + resolved argparse dest (``results_dir`` / ``systemname`` / + ``checkpoint_folder``) may be populated by an ``MLPSTORAGE_*`` env-var + fallback; we treat empty-string and ``None`` as missing. + + Emission model (D-01 / D-02 / D-04 / D-05): + * One ``error:`` line PER missing flag, using the D-02 verbatim + template. + * Immediately BELOW each error line, if the corresponding legacy + ``MLPERF_*`` env var is set AND the new ``MLPSTORAGE_*`` is not, + an adjacent ``hint:`` line using the D-05 verbatim template. + * All missing universals are checked and reported BEFORE + ``sys.exit()`` (D-01 aggregate-before-exit; T-05-06). No first- + error short-circuit. + * Exit code is ``EXIT_CODE.INVALID_ARGUMENTS`` (=2, D-03). + + ``MLPSTORAGE_CHECKPOINT_FOLDER`` has no legacy ``MLPERF_*`` + predecessor (D-08) and is intentionally absent from + ``_LEGACY_ENVVAR_MAP``, so the hint is never emitted for + checkpoint-folder — this falls out of the map lookup without a + special case. """ - errors = [] - if getattr(args, '_mlps_req_results', False): - if not getattr(args, 'results_dir', None): - errors.append( - "--results-dir/-rd is required (or set MLPERF_RESULTS_DIR)" - ) - if getattr(args, '_mlps_req_systemname', False): - if not getattr(args, 'systemname', None): - errors.append( - "--systemname/-sn is required (or set MLPERF_SYSTEMNAME)" - ) - if errors: - # Match argparse's stderr-then-exit-2 convention so existing tests - # that catch SystemExit continue to pass. - msg = "the following arguments are required: " + ", ".join(errors) - print(f"error: {msg}", file=sys.stderr) + pending = [] + for ( + marker_attr, + arg_attr, + envvar_name, + long_flag, + short_flag, + name_for_template, + ) in _UNIVERSAL_REQUIRED_SPECS: + if not getattr(args, marker_attr, False): + continue + if getattr(args, arg_attr, None): + continue + # D-02 verbatim template — kept on one physical line so the + # phase-wide grep for the exact template string finds it. + error_line = f"{long_flag}/{short_flag} is required: pass it on the command line or set MLPSTORAGE_{name_for_template}" + # D-04 / D-05: adjacent migration hint when legacy env is set and + # the new env is not. checkpoint_folder has no legacy pair, so its + # envvar_name is not a key in _LEGACY_ENVVAR_MAP and the hint stays + # None without any special-case code. + hint_line = None + legacy_name = _LEGACY_ENVVAR_MAP.get(envvar_name) + if legacy_name is not None: + legacy_val = os.environ.get(legacy_name, "") + new_val = os.environ.get(envvar_name, "") + if legacy_val and not new_val: + # D-05 verbatim template — single physical line for the + # phase-wide grep to pin the exact wording. + hint_line = f"hint: MLPERF_{name_for_template} is set but is no longer read; rename it to MLPSTORAGE_{name_for_template}" + pending.append((error_line, hint_line)) + + if pending: + for error_line, hint_line in pending: + print(f"error: {error_line}", file=sys.stderr) + if hint_line is not None: + print(hint_line, file=sys.stderr) sys.exit(EXIT_CODE.INVALID_ARGUMENTS) From ee549b7e2cf21e362aff52d4a4f17cdc48306fd1 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:50:29 -0700 Subject: [PATCH 08/45] feat(05-03): drop required=True on --checkpoint-folder, plumb marker for run - --checkpoint-folder in checkpointing_args.py no longer sets argparse-level required=True; the default is ENV_FALLBACK_CHECKPOINT_FOLDER (from MLPSTORAGE_CHECKPOINT_FOLDER env var if set) so the fallback can satisfy the requirement (D-08 / ENV-06). - parser.set_defaults(_mlps_req_checkpoint_folder=True) plumbed locally next to the arg definition inside the `command == "run"` branch so the flag and its post-parse enforcement marker stay co-located (D-09). The loud-error gate lives in cli_parser._check_universal_required_present (Plan 05-02). - HELP_MESSAGES['checkpoint_folder'] in common_args.py extended to name MLPSTORAGE_CHECKPOINT_FOLDER as the env-var fallback. - --checkpoint-folder remains defined ONLY in checkpointing_args.py (no leak to other benchmarks). --- mlpstorage_py/cli/checkpointing_args.py | 22 +++++++++++++++++++--- mlpstorage_py/cli/common_args.py | 5 ++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/mlpstorage_py/cli/checkpointing_args.py b/mlpstorage_py/cli/checkpointing_args.py index 9fcc7f43..f8d9414e 100755 --- a/mlpstorage_py/cli/checkpointing_args.py +++ b/mlpstorage_py/cli/checkpointing_args.py @@ -7,7 +7,10 @@ import sys -from mlpstorage_py.config import DEFAULT_HOSTS, EXEC_TYPE, LLM_MODELS, LLM_MODELS_CLOSED, EXIT_CODE +from mlpstorage_py.config import ( + DEFAULT_HOSTS, EXEC_TYPE, LLM_MODELS, LLM_MODELS_CLOSED, EXIT_CODE, + ENV_FALLBACK_CHECKPOINT_FOLDER, +) from mlpstorage_py.cli.common_args import ( HELP_MESSAGES, add_universal_arguments, @@ -137,14 +140,27 @@ def _add_checkpointing_core_args(parser, command): ), ) - # Checkpoint folder required for run only + # Checkpoint folder required for run only. The argparse-level required=True + # was replaced by post-parse enforcement (Phase 5 D-08 / D-09 / ENV-06) so + # the ENV_FALLBACK_CHECKPOINT_FOLDER default (sourced from + # MLPSTORAGE_CHECKPOINT_FOLDER env var) can satisfy the requirement. The + # loud-error gate lives in cli_parser._check_universal_required_present + # keyed off the _mlps_req_checkpoint_folder marker we set below. The + # --checkpoint-folder argument stays defined ONLY here (never in + # common_args.py) so no other benchmark inherits it. if command == "run": parser.add_argument( '--checkpoint-folder', '-cf', type=str, - required=True, + default=ENV_FALLBACK_CHECKPOINT_FOLDER, help=HELP_MESSAGES['checkpoint_folder'] ) + # Plumb the post-parse-gate marker locally, next to the arg definition, + # so the flag and its enforcement gate stay co-located (D-09). The + # symmetric req_checkpoint_folder kwarg on add_universal_arguments + # exists for defensive future callers; the checkpointing flow uses + # this local set_defaults. + parser.set_defaults(_mlps_req_checkpoint_folder=True) # num-checkpoints-read/write are available in all modes so closed submitters # can split write and read into two invocations (set =0 on one side) with a diff --git a/mlpstorage_py/cli/common_args.py b/mlpstorage_py/cli/common_args.py index 17ba56e5..4d8492c7 100755 --- a/mlpstorage_py/cli/common_args.py +++ b/mlpstorage_py/cli/common_args.py @@ -129,7 +129,10 @@ def add_arguments(self, actions): 'reportgen': "Generate a report from the benchmark results.", # Checkpoint folder is used for training and checkpointing - 'checkpoint_folder': "Location for checkpoint files for training or checkpointing workloads", + 'checkpoint_folder': ( + "Location for checkpoint files for training or checkpointing workloads. " + "Defaults to MLPSTORAGE_CHECKPOINT_FOLDER env var if set." + ), # Checkpointing help messages 'checkpoint_run': "The checkpoint command executes checkpoint saves and restores for a given model.", From cbbb5346f70762fa15d91c772b2ce48b0e2a4f32 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:55:05 -0700 Subject: [PATCH 09/45] test(05-05): rename test_config.py MLPERF_/DEFAULT_ symbols to MLPSTORAGE_/ENV_FALLBACK_ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rewrite TestDefaultResultsDir → TestEnvFallbackResultsDir; tempdir-fallback test now asserts _resolve_env_fallback_results_dir() returns "" when MLPSTORAGE_RESULTS_DIR is unset (D-13 tempdir retirement). - Rewrite TestDefaultSystemname → TestEnvFallbackSystemname; monkeypatches updated MLPERF_SYSTEMNAME → MLPSTORAGE_SYSTEMNAME. - Drop 'import tempfile' (no remaining consumer in the file). - All 55 tests in test_config.py pass. --- tests/unit/test_config.py | 79 +++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 71b528fa..3f095ac5 100755 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -5,11 +5,10 @@ - Environment variable handling (check_env) - Datetime string generation - Enum values and constants -- DEFAULT_RESULTS_DIR env-var override +- ENV_FALLBACK_RESULTS_DIR / ENV_FALLBACK_SYSTEMNAME env-var override """ import os -import tempfile import pytest from mlpstorage_py.config import ( @@ -305,68 +304,68 @@ def test_mpi_value(self): assert EXEC_TYPE.MPI.value == 'mpi' -class TestDefaultResultsDir: - """Tests for the DEFAULT_RESULTS_DIR constant. +class TestEnvFallbackResultsDir: + """Tests for the ENV_FALLBACK_RESULTS_DIR constant. - DEFAULT_RESULTS_DIR is set at module import time using: - os.environ.get('MLPERF_RESULTS_DIR', os.path.join(tempfile.gettempdir(), ...)) + ENV_FALLBACK_RESULTS_DIR is set at module import time using: + os.environ.get(MLPSTORAGE_RESULTS_DIR_ENVVAR, "") - Tests verify that the env-var override and the tempdir fallback both work. + Tests verify that the env-var override and the empty-string fallback both + work (Phase 5 D-13 retired the tempdir fallback — resolver returns "" when + the env var is unset). """ - def test_is_a_non_empty_string(self): - """DEFAULT_RESULTS_DIR is a non-empty string.""" - from mlpstorage_py.config import DEFAULT_RESULTS_DIR - assert isinstance(DEFAULT_RESULTS_DIR, str) - assert len(DEFAULT_RESULTS_DIR) > 0 + def test_is_a_string(self): + """ENV_FALLBACK_RESULTS_DIR is a string (empty or env-sourced).""" + from mlpstorage_py.config import ENV_FALLBACK_RESULTS_DIR + assert isinstance(ENV_FALLBACK_RESULTS_DIR, str) def test_matches_current_environment(self): - """DEFAULT_RESULTS_DIR reflects MLPERF_RESULTS_DIR if set, else the tempdir path.""" - from mlpstorage_py.config import DEFAULT_RESULTS_DIR - mlperf_env = os.environ.get('MLPERF_RESULTS_DIR') - if mlperf_env: - assert DEFAULT_RESULTS_DIR == mlperf_env + """ENV_FALLBACK_RESULTS_DIR reflects MLPSTORAGE_RESULTS_DIR if set, else empty string.""" + from mlpstorage_py.config import ENV_FALLBACK_RESULTS_DIR + mlps_env = os.environ.get('MLPSTORAGE_RESULTS_DIR') + if mlps_env: + assert ENV_FALLBACK_RESULTS_DIR == mlps_env else: - expected = os.path.join(tempfile.gettempdir(), 'mlperf_storage_results') - assert DEFAULT_RESULTS_DIR == expected + assert ENV_FALLBACK_RESULTS_DIR == "" - def test_env_var_overrides_tempdir_default(self, monkeypatch): - """When MLPERF_RESULTS_DIR is set, the resolver returns that value. + def test_env_var_overrides_default(self, monkeypatch): + """When MLPSTORAGE_RESULTS_DIR is set, the resolver returns that value. Calls the helper directly rather than reloading mlpstorage_py.config: reload would rebuild the PARAM_VALIDATION enum class, breaking identity comparisons in modules that already imported the original. """ - from mlpstorage_py.config import _resolve_default_results_dir - monkeypatch.setenv('MLPERF_RESULTS_DIR', '/custom/mlperf/results') - assert _resolve_default_results_dir() == '/custom/mlperf/results' + from mlpstorage_py.config import _resolve_env_fallback_results_dir + monkeypatch.setenv('MLPSTORAGE_RESULTS_DIR', '/custom/mlperf/results') + assert _resolve_env_fallback_results_dir() == '/custom/mlperf/results' - def test_falls_back_to_tempdir_when_env_not_set(self, monkeypatch): - """When MLPERF_RESULTS_DIR is absent, the resolver uses tempdir.""" - from mlpstorage_py.config import _resolve_default_results_dir - monkeypatch.delenv('MLPERF_RESULTS_DIR', raising=False) - expected = os.path.join(tempfile.gettempdir(), 'mlperf_storage_results') - assert _resolve_default_results_dir() == expected + def test_returns_empty_string_when_env_not_set(self, monkeypatch): + """When MLPSTORAGE_RESULTS_DIR is absent, the resolver returns empty string (D-13).""" + from mlpstorage_py.config import _resolve_env_fallback_results_dir + monkeypatch.delenv('MLPSTORAGE_RESULTS_DIR', raising=False) + assert _resolve_env_fallback_results_dir() == "" -class TestDefaultSystemname: - """Tests for the DEFAULT_SYSTEMNAME constant (LAY-04). +class TestEnvFallbackSystemname: + """Tests for the ENV_FALLBACK_SYSTEMNAME constant (LAY-04, D-12). - DEFAULT_SYSTEMNAME mirrors the DEFAULT_RESULTS_DIR pattern: reads the - MLPERF_SYSTEMNAME env var, falling back to an empty string when unset. + ENV_FALLBACK_SYSTEMNAME mirrors the ENV_FALLBACK_RESULTS_DIR pattern: reads + the MLPSTORAGE_SYSTEMNAME env var, falling back to an empty string when + unset. """ - def test_default_systemname_env_var(self, monkeypatch): - """DEFAULT_SYSTEMNAME reflects MLPERF_SYSTEMNAME env var when set, empty otherwise.""" + def test_env_fallback_systemname_env_var(self, monkeypatch): + """ENV_FALLBACK_SYSTEMNAME reflects MLPSTORAGE_SYSTEMNAME env var when set, empty otherwise.""" import mlpstorage_py.config as cfg_mod # Call the helper directly — reloading mlpstorage_py.config re-mints # PARAM_VALIDATION (and other enums), breaking enum-identity in # already-imported modules like mlpstorage_py.rules.* and corrupting # downstream integration tests. - monkeypatch.setenv('MLPERF_SYSTEMNAME', 'sys-v1') - assert cfg_mod._resolve_default_systemname() == 'sys-v1' + monkeypatch.setenv('MLPSTORAGE_SYSTEMNAME', 'sys-v1') + assert cfg_mod._resolve_env_fallback_systemname() == 'sys-v1' - monkeypatch.delenv('MLPERF_SYSTEMNAME', raising=False) - assert cfg_mod._resolve_default_systemname() == '' + monkeypatch.delenv('MLPSTORAGE_SYSTEMNAME', raising=False) + assert cfg_mod._resolve_env_fallback_systemname() == '' From 7c7d1b803cd75c4f7a55ce8213b8c3e4e3d5ff56 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:55:43 -0700 Subject: [PATCH 10/45] test(05-06): add env-var migration regression tests (TEST-12, D-17) - Pins D-02 verbatim error template and D-05 verbatim hint template - TEST-12: MLPERF_SYSTEMNAME alone still fails the ENV-04 loud-error gate - D-04 truth table for the migration hint (all four scenarios) - Covers checkpoint-folder no-legacy-hint case (D-08) - Covers training --data-dir post-YAML gate hint stanza (D-07) --- tests/unit/test_env_var_migration.py | 347 +++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 tests/unit/test_env_var_migration.py diff --git a/tests/unit/test_env_var_migration.py b/tests/unit/test_env_var_migration.py new file mode 100644 index 00000000..2a2b82d2 --- /dev/null +++ b/tests/unit/test_env_var_migration.py @@ -0,0 +1,347 @@ +""" +Env-var migration regression tests (Phase 5 D-17, TEST-12). + +This file is the negative-assertion contract for the MLPERF_* → MLPSTORAGE_* +rename. It guarantees: + + (a) TEST-12 canonical regression — setting MLPERF_SYSTEMNAME alone does + NOT satisfy the ENV-04 loud-error gate. If a future PR re-adds a + back-compat shim reading MLPERF_*, this test fails loudly. + (b) D-05 verbatim migration hint text — the exact string + ``"hint: MLPERF_ is set but is no longer read; rename it to + MLPSTORAGE_"`` appears on stderr immediately below the D-02 + error line in the correct D-04 conditions. + (c) D-04 negative condition — the hint does NOT fire when the CLI flag + was passed even with MLPERF_* set (no error → nothing to hint about). + (d) D-04 negative condition — the hint does NOT fire when both MLPERF_* + AND MLPSTORAGE_* are set (user has already migrated). + +Verbatim template strings pinned here: + D-02: "is required: pass it on the command line or set MLPSTORAGE_" + D-05: "is set but is no longer read; rename it to MLPSTORAGE_" + +Legacy MLPERF_* names appear ONLY in this file (per D-16 carveout) — they are +the negative-assertion payload. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +import pytest + +from mlpstorage_py.cli_parser import _check_universal_required_present +from mlpstorage_py.cli.training_args import validate_training_arguments + + +# ---------------------------------------------------------------------------- # +# Helpers # +# ---------------------------------------------------------------------------- # + +# Legacy env-var names the phase must NOT read. Kept as constants so future +# grep-audits that flag MLPERF_* in tests can allowlist THIS FILE cleanly. +_LEGACY_SYSTEMNAME = "MLPERF_SYSTEMNAME" +_LEGACY_RESULTS_DIR = "MLPERF_RESULTS_DIR" +_LEGACY_DATA_DIR = "MLPERF_DATA_DIR" +# NOTE: intentionally NO _LEGACY_CHECKPOINT_FOLDER — per D-08 there was never +# a MLPERF_CHECKPOINT_FOLDER; its absence from _LEGACY_ENVVAR_MAP is asserted +# by the hint-negative test below. + +_NEW_SYSTEMNAME = "MLPSTORAGE_SYSTEMNAME" +_NEW_RESULTS_DIR = "MLPSTORAGE_RESULTS_DIR" +_NEW_DATA_DIR = "MLPSTORAGE_DATA_DIR" +_NEW_CHECKPOINT_FOLDER = "MLPSTORAGE_CHECKPOINT_FOLDER" + +# All Phase-5-relevant env vars — cleared at the start of every test so no +# ambient shell state leaks in. +_ALL_ENV_NAMES = ( + _LEGACY_SYSTEMNAME, + _LEGACY_RESULTS_DIR, + _LEGACY_DATA_DIR, + _NEW_SYSTEMNAME, + _NEW_RESULTS_DIR, + _NEW_DATA_DIR, + _NEW_CHECKPOINT_FOLDER, +) + + +def _clean_env(monkeypatch): + """Clear every MLPERF_* and MLPSTORAGE_* var this file exercises.""" + for name in _ALL_ENV_NAMES: + monkeypatch.delenv(name, raising=False) + + +def _make_ns( + *, + req_results=False, + req_systemname=False, + req_checkpoint_folder=False, + results_dir="", + systemname="", + checkpoint_folder="", + mode="closed", +): + """Build a minimal argparse.Namespace shaped like a post-parse args object. + + Only the attributes ``_check_universal_required_present`` inspects are + populated — the function is a pure predicate over these markers/values. + """ + return argparse.Namespace( + mode=mode, + _mlps_req_results=req_results, + _mlps_req_systemname=req_systemname, + _mlps_req_checkpoint_folder=req_checkpoint_folder, + results_dir=results_dir, + systemname=systemname, + checkpoint_folder=checkpoint_folder, + ) + + +# ---------------------------------------------------------------------------- # +# TEST-12: MLPERF_* alone does NOT satisfy the ENV-04 gate. # +# ---------------------------------------------------------------------------- # + +class TestEnvVarMigrationCliParserGate: + """Cover the systemname / results-dir / checkpoint-folder universals whose + loud-error gate runs in ``cli_parser._check_universal_required_present``.""" + + # ------------------------------------------------------------------ (a) -- + def test_mlperf_systemname_alone_fails_env04_gate(self, monkeypatch, capsys): + """TEST-12: MLPERF_SYSTEMNAME set, MLPSTORAGE_SYSTEMNAME unset → still fails. + + Also asserts that BOTH the D-02 verbatim error line AND the D-05 + verbatim migration hint appear on stderr. + """ + _clean_env(monkeypatch) + monkeypatch.setenv(_LEGACY_SYSTEMNAME, "legacy-sys") + + ns = _make_ns(req_systemname=True, systemname="") + + with pytest.raises(SystemExit) as excinfo: + _check_universal_required_present(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + # D-02 verbatim template — this is the whole point of TEST-12. + assert ( + "--systemname/-sn is required: pass it on the command line or set MLPSTORAGE_SYSTEMNAME" + in err + ) + # D-05 verbatim template — the migration hint MUST fire here. + assert ( + "hint: MLPERF_SYSTEMNAME is set but is no longer read; rename it to MLPSTORAGE_SYSTEMNAME" + in err + ) + + # ------------------------------------------------------------------ (b) -- + def test_hint_fires_only_when_mlperf_set_and_mlpstorage_unset( + self, monkeypatch, capsys + ): + """D-04 truth table for the hint, scenario A: legacy set, new unset → hint fires.""" + _clean_env(monkeypatch) + monkeypatch.setenv(_LEGACY_SYSTEMNAME, "legacy-sys") + # MLPSTORAGE_SYSTEMNAME intentionally UNSET. + + ns = _make_ns(req_systemname=True, systemname="") + + with pytest.raises(SystemExit): + _check_universal_required_present(ns) + + err = capsys.readouterr().err + assert ( + "hint: MLPERF_SYSTEMNAME is set but is no longer read; rename it to MLPSTORAGE_SYSTEMNAME" + in err + ) + + def test_hint_does_not_fire_when_both_mlperf_and_mlpstorage_set( + self, monkeypatch, capsys + ): + """D-04 truth table for the hint, scenario B: both set, arg populated → gate passes, no output. + + When MLPSTORAGE_SYSTEMNAME is set the resolver would have populated + args.systemname, so the gate does not trigger and no error/hint is + emitted. Simulate the post-parse state where the resolver has done its + job. + """ + _clean_env(monkeypatch) + monkeypatch.setenv(_LEGACY_SYSTEMNAME, "legacy-sys") + monkeypatch.setenv(_NEW_SYSTEMNAME, "new-sys") + + # env-var default already flowed into the Namespace value. + ns = _make_ns(req_systemname=True, systemname="new-sys") + + # No SystemExit: the gate does not trigger. + _check_universal_required_present(ns) + + err = capsys.readouterr().err + # No error → no hint. Assert both are absent. + assert "error:" not in err + assert "hint:" not in err + + def test_no_hint_when_neither_env_var_is_set(self, monkeypatch, capsys): + """D-04 truth table for the hint, scenario C: neither env set → error fires, hint does NOT.""" + _clean_env(monkeypatch) + + ns = _make_ns(req_systemname=True, systemname="") + + with pytest.raises(SystemExit) as excinfo: + _check_universal_required_present(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + # Error line MUST appear (D-02). + assert ( + "--systemname/-sn is required: pass it on the command line or set MLPSTORAGE_SYSTEMNAME" + in err + ) + # Hint MUST NOT appear — no legacy state to migrate from. + assert "hint:" not in err + + # ------------------------------------------------------------------ (c) -- + def test_hint_does_not_fire_when_cli_flag_was_passed(self, monkeypatch, capsys): + """D-04: hint only appears adjacent to an error line — a satisfied gate emits nothing.""" + _clean_env(monkeypatch) + monkeypatch.setenv(_LEGACY_SYSTEMNAME, "legacy-sys") + + # The user passed --systemname on the CLI; the arg is populated. + ns = _make_ns(req_systemname=True, systemname="user-supplied-via-flag") + + # Gate does not trigger. + _check_universal_required_present(ns) + + err = capsys.readouterr().err + # Silence is the correct behavior: no error, therefore no hint. + assert "error:" not in err + assert "hint:" not in err + + # ------------------------------------------------------------------ (d) -- + def test_results_dir_migration_hint_verbatim(self, monkeypatch, capsys): + """D-05 verbatim template for the results-dir migration hint.""" + _clean_env(monkeypatch) + monkeypatch.setenv(_LEGACY_RESULTS_DIR, "/legacy/results") + + ns = _make_ns(req_results=True, results_dir="") + + with pytest.raises(SystemExit) as excinfo: + _check_universal_required_present(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + assert ( + "--results-dir/-rd is required: pass it on the command line or set MLPSTORAGE_RESULTS_DIR" + in err + ) + assert ( + "hint: MLPERF_RESULTS_DIR is set but is no longer read; rename it to MLPSTORAGE_RESULTS_DIR" + in err + ) + + def test_checkpoint_folder_has_no_migration_hint(self, monkeypatch, capsys): + """D-08: checkpoint-folder has no legacy MLPERF_* predecessor — hint never fires. + + Even if the user somehow has a MLPERF_CHECKPOINT_FOLDER set (which was + never read by any historical mlpstorage), the hint must NOT appear — + the entry is absent from ``_LEGACY_ENVVAR_MAP`` on purpose. The D-02 + error line still fires for the missing flag. + """ + _clean_env(monkeypatch) + # Simulate a user who imagined the legacy name — mlpstorage must not + # emit a rename hint, because the "legacy" name was never valid. + monkeypatch.setenv("MLPERF_CHECKPOINT_FOLDER", "/legacy/ckpt") + + ns = _make_ns(req_checkpoint_folder=True, checkpoint_folder="") + + with pytest.raises(SystemExit) as excinfo: + _check_universal_required_present(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + # D-02 error MUST still appear. + assert ( + "--checkpoint-folder/-cf is required: pass it on the command line or set MLPSTORAGE_CHECKPOINT_FOLDER" + in err + ) + # But NO migration hint — checkpoint-folder is a fresh env var. + assert "hint:" not in err + + +# ---------------------------------------------------------------------------- # +# Data-dir gate lives in training_args.validate_training_arguments (D-07) # +# — post-YAML, so it is exercised separately from the cli_parser gate above. # +# ---------------------------------------------------------------------------- # + +class TestEnvVarMigrationTrainingGate: + """Cover the training --data-dir gate. Runs post-YAML in + ``validate_training_arguments``; the migration-hint stanza there mirrors + the D-04/D-05 rules from the cli_parser gate.""" + + def test_data_dir_migration_hint_fires_verbatim(self, monkeypatch, capsys): + """D-05 verbatim template for the data-dir migration hint.""" + _clean_env(monkeypatch) + monkeypatch.setenv(_LEGACY_DATA_DIR, "/legacy/data") + + # Object protocol + run command triggers the post-YAML gate at + # training_args.py:302 when data_dir is falsy. + ns = argparse.Namespace( + command="run", + data_access_protocol="object", + data_dir=None, + o_direct=False, + ) + + with pytest.raises(SystemExit) as excinfo: + validate_training_arguments(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + # D-02 verbatim template. + assert ( + "--data-dir/-dd is required: pass it on the command line or set MLPSTORAGE_DATA_DIR" + in err + ) + # D-05 verbatim template. + assert ( + "hint: MLPERF_DATA_DIR is set but is no longer read; rename it to MLPSTORAGE_DATA_DIR" + in err + ) + + def test_data_dir_hint_absent_when_only_mlpstorage_set(self, monkeypatch, capsys): + """D-04: no legacy set → no hint, and if data_dir is populated, no error either.""" + _clean_env(monkeypatch) + monkeypatch.setenv(_NEW_DATA_DIR, "/new/data") + + # Simulate the env-var default having populated args.data_dir already. + ns = argparse.Namespace( + command="run", + data_access_protocol="object", + data_dir="/new/data", + o_direct=False, + ) + + # Gate passes; nothing printed. + validate_training_arguments(ns) + + err = capsys.readouterr().err + assert "error:" not in err + assert "hint:" not in err + + def test_data_dir_hint_absent_when_flag_passed(self, monkeypatch, capsys): + """D-04: user passed --data-dir on the CLI → no error → no hint.""" + _clean_env(monkeypatch) + monkeypatch.setenv(_LEGACY_DATA_DIR, "/legacy/data") + + ns = argparse.Namespace( + command="run", + data_access_protocol="object", + data_dir="/cli/supplied/data", + o_direct=False, + ) + + # Gate passes; nothing printed. + validate_training_arguments(ns) + + err = capsys.readouterr().err + assert "error:" not in err + assert "hint:" not in err From 7c2b245c1bf1e985659290787f07a3bf0f032a43 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:56:20 -0700 Subject: [PATCH 11/45] test(05-06): add import-direction invariant tests (D-11, D-18) - Fresh-subprocess assertion proves no import cycle exists - Structural grep asserts config.py has no 'from mlpstorage_py.rules' line - Positive-direction assertion documents rules.utils sources env-var constants from config (D-10 single-source-of-truth) --- tests/unit/test_no_import_cycles.py | 131 ++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 tests/unit/test_no_import_cycles.py diff --git a/tests/unit/test_no_import_cycles.py b/tests/unit/test_no_import_cycles.py new file mode 100644 index 00000000..16e980c8 --- /dev/null +++ b/tests/unit/test_no_import_cycles.py @@ -0,0 +1,131 @@ +""" +Import-direction invariant tests (Phase 5 D-11 / D-18). + +After Phase 5, ``mlpstorage_py.config`` owns all ``MLPSTORAGE_*_ENVVAR`` +string constants (D-10) and ``mlpstorage_py.rules.utils`` imports them from +``config`` — never the other direction. This one-way dependency prevents +circular imports and lets ``config`` stay a leaf module that any layer can +import without dragging the rules subsystem in. + +This file guards two invariants: + + (a) A fresh Python interpreter can execute + ``import mlpstorage_py.config; import mlpstorage_py.rules.utils`` + with exit code 0 — proving no cycle exists at import time. A fresh + subprocess is essential; an in-process check lies because both + modules are cached in ``sys.modules`` by the time pytest runs. + + (b) Structural grep — ``mlpstorage_py/config.py`` contains no + ``from mlpstorage_py.rules`` or ``import mlpstorage_py.rules`` line. + This catches accidental reintroduction of the reverse dependency in + code review by making the direction violation loudly visible as a + test failure rather than a runtime surprise. + +If either invariant breaks in a future PR, one of these tests fails with a +message that names the invariant, so the reviewer sees the direction +violation immediately. +""" + +from __future__ import annotations + +import pathlib +import subprocess +import sys + + +# Path to the project root — walk up from this test file to the repo root, +# then to ``mlpstorage_py/config.py``. This test file lives at +# ``tests/unit/test_no_import_cycles.py``, so root is parent x 2. +_TESTS_UNIT_DIR = pathlib.Path(__file__).resolve().parent +_REPO_ROOT = _TESTS_UNIT_DIR.parent.parent +_CONFIG_PY = _REPO_ROOT / "mlpstorage_py" / "config.py" +_RULES_UTILS_PY = _REPO_ROOT / "mlpstorage_py" / "rules" / "utils.py" + + +class TestNoImportCycles: + """Guard the config → rules.utils one-way dependency direction (D-11).""" + + def test_config_then_rules_utils_imports_cleanly_in_fresh_interpreter(self): + """A fresh subprocess must be able to import config THEN rules.utils. + + In-process checks are unreliable because both modules are already + cached in ``sys.modules`` by the time pytest runs — a cycle would + not resurface. Spawning a fresh interpreter forces the import + machinery to walk the full dependency graph from scratch. + """ + result = subprocess.run( + [ + sys.executable, + "-c", + "import mlpstorage_py.config; import mlpstorage_py.rules.utils", + ], + capture_output=True, + text=True, + timeout=30, + ) + + # Include stderr in the failure message so the diagnosis is + # immediate — the reviewer sees the ImportError / stack trace + # without needing to re-run manually. + assert result.returncode == 0, ( + "Fresh-interpreter import of config then rules.utils failed " + "(D-11 violated).\n" + f"stdout: {result.stdout!r}\n" + f"stderr: {result.stderr!r}" + ) + # Belt-and-suspenders: even if the subprocess somehow returned 0 + # with an ImportError logged, catch that too. + assert "ImportError" not in result.stderr, ( + f"ImportError surfaced in stderr: {result.stderr!r}" + ) + assert "circular" not in result.stderr.lower(), ( + f"'circular' appeared in stderr: {result.stderr!r}" + ) + + def test_config_does_not_import_from_rules(self): + """Structural check: ``config.py`` must not import from ``rules.*``. + + Grep-style assertion over the file text. Catches accidental + reintroduction of the reverse dependency in code review by making + the direction violation loudly visible as a test failure. This + assertion is what makes D-11 durable across future refactors — + the runtime import check above catches the symptom; this catches + the cause. + """ + text = _CONFIG_PY.read_text() + + # Both syntactic forms are prohibited. + assert "from mlpstorage_py.rules" not in text, ( + "mlpstorage_py/config.py contains 'from mlpstorage_py.rules' — " + "one-way dependency direction violated (D-11). config MUST NOT " + "import from rules; rules imports from config." + ) + assert "import mlpstorage_py.rules" not in text, ( + "mlpstorage_py/config.py contains 'import mlpstorage_py.rules' — " + "one-way dependency direction violated (D-11). config MUST NOT " + "import from rules; rules imports from config." + ) + + def test_rules_utils_imports_env_var_constants_from_config(self): + """Positive-direction assertion: rules.utils sources env-var names from config. + + Documents the intended direction (rules → config) and pins the + specific constants moved in Plan 05-01 per D-10. If a future + refactor tries to re-inline the env-var-name string literals in + ``rules/utils.py``, this test surfaces the duplication. + """ + text = _RULES_UTILS_PY.read_text() + + assert "from mlpstorage_py.config import" in text, ( + "mlpstorage_py/rules/utils.py must import from " + "mlpstorage_py.config (positive D-11 direction)." + ) + # The two constants moved in Plan 05-01 per D-10. + assert "MLPSTORAGE_ORGNAME_ENVVAR" in text, ( + "rules/utils.py does not reference MLPSTORAGE_ORGNAME_ENVVAR — " + "the D-10 single-source-of-truth import may have regressed." + ) + assert "MLPSTORAGE_SYSTEMNAME_ENVVAR" in text, ( + "rules/utils.py does not reference MLPSTORAGE_SYSTEMNAME_ENVVAR — " + "the D-10 single-source-of-truth import may have regressed." + ) From 7efeeb566611a481b4d0ee6471ba4a075ef0e89c Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:57:20 -0700 Subject: [PATCH 12/45] test(05-05): rename MLPERF_/DEFAULT_ symbols across CLI + run_summary tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_cli.py: rewrite the test_results_dir_required_when_req_results_true helper to patch common_args.ENV_FALLBACK_RESULTS_DIR (was DEFAULT_RESULTS_DIR) and update the docstring to reference MLPSTORAGE_RESULTS_DIR. - test_cli_parser.py: rename every MLPERF_SYSTEMNAME / MLPERF_RESULTS_DIR monkeypatch + assertion + comment to MLPSTORAGE_*, and every DEFAULT_* attribute patch / _resolve_default_* call to ENV_FALLBACK_* / _resolve_env_fallback_*. Update the results-dir-env-var-satisfies test docstring to reflect D-13 (tempdir fallback retired; empty-string fallback now). - test_run_summary.py: rename test_banner_environment_includes_mlperf_systemname → *_mlpstorage_systemname; add sibling test asserting the Environment section contains all five MLPSTORAGE_* rows added by Plan 05-04 (RESULTS_DIR, SYSTEMNAME, ORGNAME, DATA_DIR, CHECKPOINT_FOLDER). - 204 passed / 1 skipped across the three files. --- tests/unit/test_cli.py | 8 +-- tests/unit/test_cli_parser.py | 106 +++++++++++++++++---------------- tests/unit/test_run_summary.py | 40 ++++++++++--- 3 files changed, 91 insertions(+), 63 deletions(-) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 21bda349..67698cd1 100755 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -125,15 +125,15 @@ def test_results_dir_required_when_req_results_true(self, parser, monkeypatch): Post-CR-02 fix: the requirement is enforced at the ``validate_args`` (post-parse) layer rather than via argparse ``required=True``, so the env-var-sourced default can satisfy it. - With ``MLPERF_RESULTS_DIR`` unset AND ``DEFAULT_RESULTS_DIR`` also - emptied (to simulate the "no env var, no fallback" worst case), + With ``MLPSTORAGE_RESULTS_DIR`` unset AND ``ENV_FALLBACK_RESULTS_DIR`` + also emptied (to simulate the "no env var, no fallback" worst case), the validator must error out via SystemExit. """ from mlpstorage_py.cli_parser import _check_universal_required_present - # Force DEFAULT_RESULTS_DIR to '' so the resolved value is empty + # Force ENV_FALLBACK_RESULTS_DIR to '' so the resolved value is empty # even though argparse no longer demands the CLI flag. import mlpstorage_py.cli.common_args as common_args_mod - monkeypatch.setattr(common_args_mod, 'DEFAULT_RESULTS_DIR', '') + monkeypatch.setattr(common_args_mod, 'ENV_FALLBACK_RESULTS_DIR', '') add_universal_arguments(parser, req_results=True) args = parser.parse_args([]) # no error from argparse itself now assert getattr(args, '_mlps_req_results', False) is True, ( diff --git a/tests/unit/test_cli_parser.py b/tests/unit/test_cli_parser.py index db779e7c..eface108 100755 --- a/tests/unit/test_cli_parser.py +++ b/tests/unit/test_cli_parser.py @@ -392,11 +392,11 @@ def test_no_file_object_consolidation_needed(self): # ===================================================================== -# 6. --systemname / MLPERF_SYSTEMNAME plumbing (LAY-04, D-10) +# 6. --systemname / MLPSTORAGE_SYSTEMNAME plumbing (LAY-04, D-10) # ===================================================================== class TestSystemname: - """Tests for the --systemname flag and MLPERF_SYSTEMNAME env-var plumbing. + """Tests for the --systemname flag and MLPSTORAGE_SYSTEMNAME env-var plumbing. Per CONTEXT.md D-10, --systemname is required on every emitting subcommand: training {datagen, run, configview, datasize}, checkpointing {datagen, run, @@ -462,7 +462,7 @@ class TestSystemname: ) def test_systemname_on_emitting_commands(self, label, argv_tail, monkeypatch): """Every emitting subcommand accepts --systemname and binds it to args.systemname.""" - monkeypatch.delenv('MLPERF_SYSTEMNAME', raising=False) + monkeypatch.delenv('MLPSTORAGE_SYSTEMNAME', raising=False) full = ['mlpstorage'] + argv_tail + ['--systemname', 'sys-v1'] with patch('sys.argv', full): args = parse_arguments() @@ -481,10 +481,11 @@ def test_empty_systemname_errors(self, label, argv_tail, monkeypatch): Post CR-02 fix: ``--systemname`` is no longer ``required=True`` at the argparse layer (that would silently neuter the - ``MLPERF_SYSTEMNAME`` env-var fallback per D-10). Instead, the post- - parse validator checks that the resolved value is non-empty; with - neither the CLI flag nor the env var supplied, ``DEFAULT_SYSTEMNAME`` - resolves to ``""`` and the validator errors out. + ``MLPSTORAGE_SYSTEMNAME`` env-var fallback per D-10). Instead, the + post-parse validator checks that the resolved value is non-empty; + with neither the CLI flag nor the env var supplied, + ``ENV_FALLBACK_SYSTEMNAME`` resolves to ``""`` and the validator + errors out. ``reports reportgen`` is intentionally excluded: reportgen makes ``--systemname`` OPTIONAL because omitting it aggregates a global @@ -499,20 +500,20 @@ def test_empty_systemname_errors(self, label, argv_tail, monkeypatch): "no-systemname aggregation contract." ) # Ensure env var is unset so the default falls back to '' (empty). - monkeypatch.delenv('MLPERF_SYSTEMNAME', raising=False) + monkeypatch.delenv('MLPSTORAGE_SYSTEMNAME', raising=False) full = ['mlpstorage'] + argv_tail # no --systemname with patch('sys.argv', full): with pytest.raises(SystemExit): parse_arguments() - # ---- CR-02: MLPERF_SYSTEMNAME / MLPERF_RESULTS_DIR env-var fallback ---- + # ---- CR-02: MLPSTORAGE_SYSTEMNAME / MLPSTORAGE_RESULTS_DIR env-var fallback ---- # # The reviewer flagged that ``required=True`` + ``default=`` is # contradictory in argparse — ``required=True`` checks the CLI tokens, # not the resolved value, so the env-var defaults were dead on every # emitting subcommand. These tests pin the post-fix behavior: when - # ``MLPERF_SYSTEMNAME`` / ``MLPERF_RESULTS_DIR`` is set, the CLI flag - # is no longer mandatory. + # ``MLPSTORAGE_SYSTEMNAME`` / ``MLPSTORAGE_RESULTS_DIR`` is set, the CLI + # flag is no longer mandatory. @pytest.mark.parametrize( "label,argv_tail", @@ -522,25 +523,26 @@ def test_empty_systemname_errors(self, label, argv_tail, monkeypatch): def test_systemname_env_var_satisfies_requirement( self, label, argv_tail, monkeypatch, ): - """If MLPERF_SYSTEMNAME is set, ``--systemname`` may be omitted on the CLI. + """If MLPSTORAGE_SYSTEMNAME is set, ``--systemname`` may be omitted on the CLI. Pre-fix this raised SystemExit because argparse's ``required=True`` ignored the env-var-sourced default. Post-fix, the resolved - ``DEFAULT_SYSTEMNAME`` (sourced from the env var) satisfies the - requirement and the parsed namespace carries that value. + ``ENV_FALLBACK_SYSTEMNAME`` (sourced from the env var) satisfies + the requirement and the parsed namespace carries that value. """ - monkeypatch.setenv('MLPERF_SYSTEMNAME', 'env-sys-v1') - # Patch DEFAULT_SYSTEMNAME in place — do NOT reload mlpstorage_py.config. - # Reloading config re-mints PARAM_VALIDATION (and other enums), breaking - # `enum_instance in [enum_class.MEMBER, ...]` checks in any module that - # already imported them (notably mlpstorage_py.rules.*). Downstream CLI - # builders captured DEFAULT_SYSTEMNAME by name at import time, so reload - # them so they pick up the patched value. + monkeypatch.setenv('MLPSTORAGE_SYSTEMNAME', 'env-sys-v1') + # Patch ENV_FALLBACK_SYSTEMNAME in place — do NOT reload + # mlpstorage_py.config. Reloading config re-mints PARAM_VALIDATION + # (and other enums), breaking `enum_instance in [enum_class.MEMBER, + # ...]` checks in any module that already imported them (notably + # mlpstorage_py.rules.*). Downstream CLI builders captured + # ENV_FALLBACK_SYSTEMNAME by name at import time, so reload them so + # they pick up the patched value. import importlib import mlpstorage_py.config as cfg_mod import mlpstorage_py.cli.common_args as common_args_mod - saved_systemname = cfg_mod.DEFAULT_SYSTEMNAME - cfg_mod.DEFAULT_SYSTEMNAME = 'env-sys-v1' + saved_systemname = cfg_mod.ENV_FALLBACK_SYSTEMNAME + cfg_mod.ENV_FALLBACK_SYSTEMNAME = 'env-sys-v1' importlib.reload(common_args_mod) import mlpstorage_py.cli as cli_mod importlib.reload(cli_mod) @@ -551,37 +553,37 @@ def test_systemname_env_var_satisfies_requirement( with patch('sys.argv', full): args = cli_parser_mod.parse_arguments() assert getattr(args, 'systemname', None) == 'env-sys-v1', ( - f"{label}: MLPERF_SYSTEMNAME env var must satisfy --systemname " + f"{label}: MLPSTORAGE_SYSTEMNAME env var must satisfy --systemname " f"requirement; got {getattr(args, 'systemname', None)!r}" ) finally: - monkeypatch.delenv('MLPERF_SYSTEMNAME', raising=False) - cfg_mod.DEFAULT_SYSTEMNAME = saved_systemname + monkeypatch.delenv('MLPSTORAGE_SYSTEMNAME', raising=False) + cfg_mod.ENV_FALLBACK_SYSTEMNAME = saved_systemname importlib.reload(common_args_mod) importlib.reload(cli_mod) importlib.reload(cli_parser_mod) def test_results_dir_env_var_satisfies_requirement(self, monkeypatch): - """If MLPERF_RESULTS_DIR is set, ``--results-dir`` may be omitted on the CLI. + """If MLPSTORAGE_RESULTS_DIR is set, ``--results-dir`` may be omitted on the CLI. - DEFAULT_RESULTS_DIR has a non-empty tempdir fallback even without - the env var, but the contractual D-10 promise is that - MLPERF_RESULTS_DIR is honored as a default on every emitting - subcommand. Pre-fix, ``required=True`` ignored the env var entirely. + Post D-13, ``ENV_FALLBACK_RESULTS_DIR`` returns ``""`` when the env + var is unset (the tempdir fallback was retired). The contractual + D-10 promise remains: ``MLPSTORAGE_RESULTS_DIR`` is honored as a + default on every emitting subcommand. """ - monkeypatch.setenv('MLPERF_RESULTS_DIR', '/env/results') - monkeypatch.setenv('MLPERF_SYSTEMNAME', 'env-sys') # so systemname check passes - # Patch DEFAULT_RESULTS_DIR + DEFAULT_SYSTEMNAME in place — do NOT - # reload mlpstorage_py.config (see sibling test for the enum-identity - # rationale). Reload the downstream CLI builders so the patched values - # propagate into argparse defaults. + monkeypatch.setenv('MLPSTORAGE_RESULTS_DIR', '/env/results') + monkeypatch.setenv('MLPSTORAGE_SYSTEMNAME', 'env-sys') # so systemname check passes + # Patch ENV_FALLBACK_RESULTS_DIR + ENV_FALLBACK_SYSTEMNAME in place + # — do NOT reload mlpstorage_py.config (see sibling test for the + # enum-identity rationale). Reload the downstream CLI builders so + # the patched values propagate into argparse defaults. import importlib import mlpstorage_py.config as cfg_mod import mlpstorage_py.cli.common_args as common_args_mod - saved_results_dir = cfg_mod.DEFAULT_RESULTS_DIR - saved_systemname = cfg_mod.DEFAULT_SYSTEMNAME - cfg_mod.DEFAULT_RESULTS_DIR = '/env/results' - cfg_mod.DEFAULT_SYSTEMNAME = 'env-sys' + saved_results_dir = cfg_mod.ENV_FALLBACK_RESULTS_DIR + saved_systemname = cfg_mod.ENV_FALLBACK_SYSTEMNAME + cfg_mod.ENV_FALLBACK_RESULTS_DIR = '/env/results' + cfg_mod.ENV_FALLBACK_SYSTEMNAME = 'env-sys' importlib.reload(common_args_mod) import mlpstorage_py.cli as cli_mod importlib.reload(cli_mod) @@ -598,21 +600,21 @@ def test_results_dir_env_var_satisfies_requirement(self, monkeypatch): with patch('sys.argv', argv): args = cli_parser_mod.parse_arguments() assert args.results_dir == '/env/results', ( - f"MLPERF_RESULTS_DIR env var must satisfy --results-dir " + f"MLPSTORAGE_RESULTS_DIR env var must satisfy --results-dir " f"requirement; got {args.results_dir!r}" ) finally: - monkeypatch.delenv('MLPERF_RESULTS_DIR', raising=False) - monkeypatch.delenv('MLPERF_SYSTEMNAME', raising=False) - cfg_mod.DEFAULT_RESULTS_DIR = saved_results_dir - cfg_mod.DEFAULT_SYSTEMNAME = saved_systemname + monkeypatch.delenv('MLPSTORAGE_RESULTS_DIR', raising=False) + monkeypatch.delenv('MLPSTORAGE_SYSTEMNAME', raising=False) + cfg_mod.ENV_FALLBACK_RESULTS_DIR = saved_results_dir + cfg_mod.ENV_FALLBACK_SYSTEMNAME = saved_systemname importlib.reload(common_args_mod) importlib.reload(cli_mod) importlib.reload(cli_parser_mod) def test_lockfile_does_not_require_systemname(self, monkeypatch): """Pure utility subcommands (lockfile) parse without --systemname.""" - monkeypatch.delenv('MLPERF_SYSTEMNAME', raising=False) + monkeypatch.delenv('MLPSTORAGE_SYSTEMNAME', raising=False) with patch('sys.argv', ['mlpstorage', 'lockfile', 'generate', '-o', '/tmp/x', '--results-dir', '/r']): args = parse_arguments() @@ -620,13 +622,13 @@ def test_lockfile_does_not_require_systemname(self, monkeypatch): def test_init_does_not_require_systemname(self, monkeypatch): """`mlpstorage init` parses without --systemname (init is bootstrap, not emitting).""" - monkeypatch.delenv('MLPERF_SYSTEMNAME', raising=False) + monkeypatch.delenv('MLPSTORAGE_SYSTEMNAME', raising=False) with patch('sys.argv', ['mlpstorage', 'init', 'Acme', '/tmp/r']): args = parse_arguments() assert args.mode == 'init' def test_systemname_flag_default_reflects_env_var(self, monkeypatch): - """The argparse default for --systemname reflects DEFAULT_SYSTEMNAME (which reads MLPERF_SYSTEMNAME). + """The argparse default for --systemname reflects ENV_FALLBACK_SYSTEMNAME (which reads MLPSTORAGE_SYSTEMNAME). Per CONTEXT.md D-10 / plan Task 1, --systemname is `required=True` on emitting commands — argparse demands the flag on CLI even when the @@ -635,9 +637,9 @@ def test_systemname_flag_default_reflects_env_var(self, monkeypatch): config-module constant; tests of the empty-systemname raise live in the slice-3 / slice-4 generate_output_location path, not the parser. """ - monkeypatch.setenv('MLPERF_SYSTEMNAME', 'env-sys') + monkeypatch.setenv('MLPSTORAGE_SYSTEMNAME', 'env-sys') # Call the env-resolver helper directly — reloading mlpstorage_py.config # re-mints PARAM_VALIDATION and corrupts enum-identity in downstream - # tests (see _resolve_default_systemname docstring in config.py). + # tests (see _resolve_env_fallback_systemname docstring in config.py). import mlpstorage_py.config as cfg_mod - assert cfg_mod._resolve_default_systemname() == 'env-sys' + assert cfg_mod._resolve_env_fallback_systemname() == 'env-sys' diff --git a/tests/unit/test_run_summary.py b/tests/unit/test_run_summary.py index 2b0e0481..fb27d87b 100644 --- a/tests/unit/test_run_summary.py +++ b/tests/unit/test_run_summary.py @@ -266,28 +266,54 @@ def test_banner_shows_systemname(self, mock_logger): ) @patch('mlpstorage_py.run_summary.logger') - def test_banner_environment_includes_mlperf_systemname(self, mock_logger, - monkeypatch): - """Environment section lists the MLPERF_SYSTEMNAME env var alongside - the existing MLPERF_RESULTS_DIR row. + def test_banner_environment_includes_mlpstorage_systemname(self, mock_logger, + monkeypatch): + """Environment section lists the MLPSTORAGE_SYSTEMNAME env var alongside + the existing MLPSTORAGE_RESULTS_DIR row. """ from mlpstorage_py.run_summary import print_run_summary - monkeypatch.setenv('MLPERF_SYSTEMNAME', 'env-sys-v1') + monkeypatch.setenv('MLPSTORAGE_SYSTEMNAME', 'env-sys-v1') args = _make_args(orgname='Acme', systemname='env-sys-v1', mode='closed', results_dir='/r') print_run_summary(args) output = _joined_status_calls(mock_logger) - assert 'MLPERF_SYSTEMNAME' in output, ( - f"Expected MLPERF_SYSTEMNAME env-var row in Environment section; " + assert 'MLPSTORAGE_SYSTEMNAME' in output, ( + f"Expected MLPSTORAGE_SYSTEMNAME env-var row in Environment section; " f"got: {output!r}" ) assert 'env-sys-v1' in output, ( f"Expected env-var value 'env-sys-v1' in output; got: {output!r}" ) + @patch('mlpstorage_py.run_summary.logger') + def test_banner_environment_includes_all_mlpstorage_rows(self, mock_logger, + monkeypatch): + """Environment section contains all five MLPSTORAGE_* env-var rows + added by Plan 05-04 (RESULTS_DIR, SYSTEMNAME, ORGNAME, DATA_DIR, + CHECKPOINT_FOLDER). + """ + from mlpstorage_py.run_summary import print_run_summary + + args = _make_args(orgname='Acme', systemname='sys-v1', + mode='closed', results_dir='/r') + print_run_summary(args) + + output = _joined_status_calls(mock_logger) + for row_label in ( + 'MLPSTORAGE_RESULTS_DIR', + 'MLPSTORAGE_SYSTEMNAME', + 'MLPSTORAGE_ORGNAME', + 'MLPSTORAGE_DATA_DIR', + 'MLPSTORAGE_CHECKPOINT_FOLDER', + ): + assert row_label in output, ( + f"Expected {row_label} row in Environment section; " + f"got: {output!r}" + ) + class TestOutputOnlyDenylist: """Output-only knobs (quiet/debug/verbose/stream_log_level) never appear as their own rows.""" From 5861b9cfe576be77112a21ca829b55d2a20e6a76 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:57:24 -0700 Subject: [PATCH 13/45] test(05-06): add loud-error contract tests (TEST-13, ENV-06, D-19) - Pins D-02 verbatim error template for all four required universals: --results-dir, --systemname, --data-dir, --checkpoint-folder - Asserts BOTH the flag string AND the MLPSTORAGE_* env-var string appear on ONE 'error:' line per D-02 - ENV-06 canonical test drives checkpoint-folder end-to-end through parse_arguments with a patched sys.argv - D-01 aggregate-before-exit: three missing universals produce three error lines and one sys.exit(2) call --- tests/unit/test_env_var_loud_errors.py | 280 +++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 tests/unit/test_env_var_loud_errors.py diff --git a/tests/unit/test_env_var_loud_errors.py b/tests/unit/test_env_var_loud_errors.py new file mode 100644 index 00000000..b2a6c40d --- /dev/null +++ b/tests/unit/test_env_var_loud_errors.py @@ -0,0 +1,280 @@ +""" +Loud-error contract tests for Phase 5 required universals (TEST-13, ENV-06, D-19). + +This file is the primary defender of the D-02 verbatim error template. When a +required universal is missing at post-parse time, mlpstorage MUST exit 2 with +a single ``error:`` line that names BOTH the CLI flag (long/short) AND the +corresponding ``MLPSTORAGE_*`` env var. TEST-13 pins this for the three +cli_parser-gate universals; ENV-06 pins it for ``--checkpoint-folder``. + +Covers four universals: + 1. ``--results-dir/-rd`` ↔ ``MLPSTORAGE_RESULTS_DIR`` (TEST-13) + 2. ``--systemname/-sn`` ↔ ``MLPSTORAGE_SYSTEMNAME`` (TEST-13) + 3. ``--data-dir/-dd`` ↔ ``MLPSTORAGE_DATA_DIR`` (TEST-13, training post-YAML gate) + 4. ``--checkpoint-folder/-cf`` ↔ ``MLPSTORAGE_CHECKPOINT_FOLDER`` (ENV-06) + +Contract invariants asserted in every test: + - D-01: One ``error:`` line per missing flag; multiple universals aggregate + BEFORE ``sys.exit`` (no first-error short-circuit). + - D-02: Verbatim template + ``"--{long}/-{short} is required: pass it on the command line or set MLPSTORAGE_{NAME}"`` + — pinned by substring match so future paraphrases fail loudly. + - D-03: Exit code is ``EXIT_CODE.INVALID_ARGUMENTS`` (= 2). + +Design notes: + - The three cli_parser gates are unit-tested in-process via + ``_check_universal_required_present`` with a hand-built Namespace. This is + the fastest and most focused surface — no subparser wiring noise. + - The data-dir gate lives in ``training_args.validate_training_arguments`` + (D-07: post-YAML). Tested via a direct call with a stubbed Namespace. + - The checkpoint-folder gate is exercised end-to-end through + ``parse_arguments`` (patching ``sys.argv``) because the + ``_mlps_req_checkpoint_folder=True`` marker is set inside the + ``checkpointing_args`` subparser build — reproducing that plumbing + manually would just duplicate the source under test. +""" + +from __future__ import annotations + +import argparse +import sys +from unittest.mock import patch + +import pytest + +from mlpstorage_py.cli_parser import ( + _check_universal_required_present, + parse_arguments, +) +from mlpstorage_py.cli.training_args import validate_training_arguments + + +# ---------------------------------------------------------------------------- # +# Helpers # +# ---------------------------------------------------------------------------- # + +# Env vars this file exercises — cleared before every test so ambient shell +# state does not leak in. +_ALL_ENV_NAMES = ( + "MLPERF_SYSTEMNAME", + "MLPERF_RESULTS_DIR", + "MLPERF_DATA_DIR", + "MLPSTORAGE_SYSTEMNAME", + "MLPSTORAGE_RESULTS_DIR", + "MLPSTORAGE_DATA_DIR", + "MLPSTORAGE_CHECKPOINT_FOLDER", +) + + +def _clean_env(monkeypatch): + for name in _ALL_ENV_NAMES: + monkeypatch.delenv(name, raising=False) + + +def _make_ns( + *, + req_results=False, + req_systemname=False, + req_checkpoint_folder=False, + results_dir="", + systemname="", + checkpoint_folder="", + mode="closed", +): + """Build a Namespace shaped for ``_check_universal_required_present``.""" + return argparse.Namespace( + mode=mode, + _mlps_req_results=req_results, + _mlps_req_systemname=req_systemname, + _mlps_req_checkpoint_folder=req_checkpoint_folder, + results_dir=results_dir, + systemname=systemname, + checkpoint_folder=checkpoint_folder, + ) + + +def _one_line_contains_both(stderr: str, needle_a: str, needle_b: str) -> bool: + """True iff at least one line in ``stderr`` contains BOTH substrings. + + D-02 is a single-line contract: the flag string and the env-var string + must appear on the SAME physical line so users grepping for either one + see the complete actionable message. + """ + return any(needle_a in line and needle_b in line for line in stderr.splitlines()) + + +# ---------------------------------------------------------------------------- # +# TEST-13: --results-dir / MLPSTORAGE_RESULTS_DIR # +# ---------------------------------------------------------------------------- # + +class TestResultsDirLoudError: + """D-02 verbatim template + single-line contract for the results-dir gate.""" + + def test_missing_results_dir_emits_verbatim_d02_template( + self, monkeypatch, capsys + ): + _clean_env(monkeypatch) + ns = _make_ns(req_results=True, results_dir="") + + with pytest.raises(SystemExit) as excinfo: + _check_universal_required_present(ns) + # D-03: exit code 2. + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + # D-02 verbatim template — full string, pinned. + assert ( + "error: --results-dir/-rd is required: pass it on the command line or set MLPSTORAGE_RESULTS_DIR" + in err + ) + # Single-line invariant: flag AND env var on the SAME line. + assert _one_line_contains_both( + err, "--results-dir/-rd", "MLPSTORAGE_RESULTS_DIR" + ) + + +# ---------------------------------------------------------------------------- # +# TEST-13: --systemname / MLPSTORAGE_SYSTEMNAME # +# ---------------------------------------------------------------------------- # + +class TestSystemnameLoudError: + """D-02 verbatim template + single-line contract for the systemname gate.""" + + def test_missing_systemname_emits_verbatim_d02_template( + self, monkeypatch, capsys + ): + _clean_env(monkeypatch) + ns = _make_ns(req_systemname=True, systemname="") + + with pytest.raises(SystemExit) as excinfo: + _check_universal_required_present(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + assert ( + "error: --systemname/-sn is required: pass it on the command line or set MLPSTORAGE_SYSTEMNAME" + in err + ) + assert _one_line_contains_both(err, "--systemname/-sn", "MLPSTORAGE_SYSTEMNAME") + + +# ---------------------------------------------------------------------------- # +# TEST-13: --data-dir / MLPSTORAGE_DATA_DIR (post-YAML training gate, D-07) # +# ---------------------------------------------------------------------------- # + +class TestDataDirLoudError: + """D-02 verbatim template + single-line contract for the training --data-dir + gate. The gate lives in ``validate_training_arguments`` and runs AFTER + YAML config merge (D-07) so ``--config-file`` can supply ``data_dir``.""" + + def test_missing_data_dir_emits_verbatim_d02_template( + self, monkeypatch, capsys + ): + _clean_env(monkeypatch) + + # Object protocol + run command is the D-07 gate trigger. + ns = argparse.Namespace( + command="run", + data_access_protocol="object", + data_dir=None, + o_direct=False, + ) + + with pytest.raises(SystemExit) as excinfo: + validate_training_arguments(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + assert ( + "error: --data-dir/-dd is required: pass it on the command line or set MLPSTORAGE_DATA_DIR" + in err + ) + assert _one_line_contains_both(err, "--data-dir/-dd", "MLPSTORAGE_DATA_DIR") + + +# ---------------------------------------------------------------------------- # +# ENV-06: --checkpoint-folder / MLPSTORAGE_CHECKPOINT_FOLDER # +# ---------------------------------------------------------------------------- # + +class TestCheckpointFolderLoudError: + """D-02 verbatim template + single-line contract for the checkpoint-folder + gate (ENV-06). Exercised end-to-end through ``parse_arguments`` because + the ``_mlps_req_checkpoint_folder`` marker is set inside the checkpointing + subparser build (checkpointing_args.py per D-08 / D-09).""" + + def test_missing_checkpoint_folder_emits_verbatim_d02_template( + self, monkeypatch, capsys + ): + _clean_env(monkeypatch) + + # Bare 'checkpointing run' with every OTHER required flag supplied but + # NOT --checkpoint-folder. --results-dir and --systemname are supplied + # so ONLY the checkpoint-folder gate can fire. + argv = [ + "mlpstorage", "closed", "checkpointing", "run", + "-cm", "1024", + "-m", "llama3-8b", + "-np", "8", + "-rd", "/tmp", + "-sn", "sys-v1", + "file", + ] + + with patch("sys.argv", argv): + with pytest.raises(SystemExit) as excinfo: + parse_arguments() + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + assert ( + "error: --checkpoint-folder/-cf is required: pass it on the command line or set MLPSTORAGE_CHECKPOINT_FOLDER" + in err + ) + assert _one_line_contains_both( + err, "--checkpoint-folder/-cf", "MLPSTORAGE_CHECKPOINT_FOLDER" + ) + + +# ---------------------------------------------------------------------------- # +# D-01: aggregate-before-exit — multiple missing universals produce multiple # +# error lines, and sys.exit is called ONCE at the end. # +# ---------------------------------------------------------------------------- # + +class TestMultipleMissingUniversalsAggregate: + """D-01: no first-error short-circuit. All missing universals reported.""" + + def test_three_missing_universals_produce_three_error_lines( + self, monkeypatch, capsys + ): + _clean_env(monkeypatch) + # All three cli_parser-gate universals required and empty. + ns = _make_ns( + req_results=True, + req_systemname=True, + req_checkpoint_folder=True, + results_dir="", + systemname="", + checkpoint_folder="", + ) + + with pytest.raises(SystemExit) as excinfo: + _check_universal_required_present(ns) + assert excinfo.value.code == 2 + + err = capsys.readouterr().err + # Three error lines — aggregate emission (D-01). + assert err.count("error:") == 3 + + # Each of the three D-02 verbatim templates appears. + assert ( + "--results-dir/-rd is required: pass it on the command line or set MLPSTORAGE_RESULTS_DIR" + in err + ) + assert ( + "--systemname/-sn is required: pass it on the command line or set MLPSTORAGE_SYSTEMNAME" + in err + ) + assert ( + "--checkpoint-folder/-cf is required: pass it on the command line or set MLPSTORAGE_CHECKPOINT_FOLDER" + in err + ) From 484d10f88c40bb488d5e07aa0c907abad0ea4bc5 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Thu, 2 Jul 2026 17:57:57 -0700 Subject: [PATCH 14/45] test(05-05): delete test_main_warnings.py (D-15 / TEST-16) The tempdir-fallback warning that this file exercised is structurally unreachable after Plan 05-01 deleted the fallback path and the dead-code warn block. D-15 mandates first-class deletion (no skip-marker stub). Also remove the corresponding row from tests/README.md and update the neighboring test_config.py row to reference the new ENV_FALLBACK_RESULTS_DIR / ENV_FALLBACK_SYSTEMNAME constants (D-12). --- tests/README.md | 3 +- tests/unit/test_main_warnings.py | 144 ------------------------------- 2 files changed, 1 insertion(+), 146 deletions(-) delete mode 100644 tests/unit/test_main_warnings.py diff --git a/tests/README.md b/tests/README.md index 616721c7..961dbc86 100644 --- a/tests/README.md +++ b/tests/README.md @@ -255,9 +255,8 @@ pytest tests/unit/test_benchmarks_kvcache.py -v | `test_cli_kvcache.py` | CLI argument parsing — KV cache model and cache configuration | | `test_cli_vectordb.py` | CLI argument parsing — VectorDB run/datagen subcommands | | `test_cluster_collector.py` | Cluster metric collection | -| `test_config.py` | Config module, env var handling, `DEFAULT_RESULTS_DIR` env-var override | +| `test_config.py` | Config module, env var handling, `ENV_FALLBACK_RESULTS_DIR` / `ENV_FALLBACK_SYSTEMNAME` env-var override | | `test_dlio_object_storage.py` | `DLIOBenchmark._apply_object_storage_params()` — `.env` loading, param injection, error cases | -| `test_main_warnings.py` | `run_benchmark()` tempdir warning — fires/suppresses correctly | | `test_dependency_check.py` | Dependency checking logic | | `test_environment.py` | Environment detection and validation | | `test_history.py` | `HistoryTracker` — run history file management | diff --git a/tests/unit/test_main_warnings.py b/tests/unit/test_main_warnings.py deleted file mode 100644 index 37130610..00000000 --- a/tests/unit/test_main_warnings.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Tests for warning/info messages emitted by mlpstorage_py.main.run_benchmark(). - -Changes under test: - - A warning is logged when results_dir defaults to the system temp directory - and MLPERF_RESULTS_DIR is not set in the environment. - - No warning when the user explicitly passes --results-dir. - - No warning when MLPERF_RESULTS_DIR is set (the default already reflects the - env var, so the user has expressed a preference). -""" - -import os -import tempfile -from argparse import Namespace -from unittest.mock import MagicMock, patch - -import pytest - -from mlpstorage_py.config import EXIT_CODE - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _make_args(results_dir=None): - """Return a minimal Namespace accepted by run_benchmark().""" - from mlpstorage_py.config import DEFAULT_RESULTS_DIR - return Namespace( - benchmark='training', - results_dir=results_dir if results_dir is not None else DEFAULT_RESULTS_DIR, - verify_lockfile=None, # skip lockfile validation branch - skip_validation=True, # skip environment validation branch - dry_run=False, - ) - - -def _mock_benchmark(): - """Return a mock benchmark whose run() returns SUCCESS.""" - b = MagicMock() - b.run.return_value = EXIT_CODE.SUCCESS - return b - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - -class TestResultsDirWarning: - """run_benchmark() warns when results land in the system temp directory.""" - - @patch('mlpstorage_py.benchmarks.TrainingBenchmark') - @patch('mlpstorage_py.main.logger') - def test_warning_emitted_when_using_tempdir_default( - self, mock_logger, mock_training_cls, monkeypatch - ): - """Warning fires when results_dir == DEFAULT_RESULTS_DIR and env var unset.""" - from mlpstorage_py.main import run_benchmark - from mlpstorage_py.config import DEFAULT_RESULTS_DIR - - monkeypatch.delenv('MLPERF_RESULTS_DIR', raising=False) - mock_training_cls.return_value = _mock_benchmark() - - args = _make_args(DEFAULT_RESULTS_DIR) - run_benchmark(args, '20260427_120000') - - # At least one warning call should mention the temp directory - assert mock_logger.warning.called, "Expected logger.warning to be called" - warning_text = ' '.join( - str(c) for c in mock_logger.warning.call_args_list - ).lower() - assert 'temp' in warning_text or 'tmp' in warning_text, ( - f"Expected temp-dir mention in warning, got: {warning_text}" - ) - - @patch('mlpstorage_py.benchmarks.TrainingBenchmark') - @patch('mlpstorage_py.main.logger') - def test_warning_mentions_results_dir_flag( - self, mock_logger, mock_training_cls, monkeypatch - ): - """Warning text tells the user about --results-dir and MLPERF_RESULTS_DIR.""" - from mlpstorage_py.main import run_benchmark - from mlpstorage_py.config import DEFAULT_RESULTS_DIR - - monkeypatch.delenv('MLPERF_RESULTS_DIR', raising=False) - mock_training_cls.return_value = _mock_benchmark() - - run_benchmark(_make_args(DEFAULT_RESULTS_DIR), '20260427_120000') - - warning_text = ' '.join( - str(c) for c in mock_logger.warning.call_args_list - ) - assert 'results-dir' in warning_text or '--results-dir' in warning_text, ( - "Warning should tell users about the --results-dir flag" - ) - assert 'MLPERF_RESULTS_DIR' in warning_text, ( - "Warning should tell users about the MLPERF_RESULTS_DIR env var" - ) - - @patch('mlpstorage_py.benchmarks.TrainingBenchmark') - @patch('mlpstorage_py.main.logger') - def test_no_tempdir_warning_when_results_dir_explicitly_set( - self, mock_logger, mock_training_cls, monkeypatch - ): - """No tempdir warning when the user supplies an explicit results directory.""" - from mlpstorage_py.main import run_benchmark - - monkeypatch.delenv('MLPERF_RESULTS_DIR', raising=False) - mock_training_cls.return_value = _mock_benchmark() - - run_benchmark(_make_args('/explicit/results/path'), '20260427_120000') - - # Inspect all warning calls — none should be about the temp directory - for call in mock_logger.warning.call_args_list: - text = str(call).lower() - assert 'temp directory' not in text and 'mlperf_results_dir' not in text, ( - f"Unexpected tempdir warning when results_dir was explicit: {call}" - ) - - @patch('mlpstorage_py.benchmarks.TrainingBenchmark') - @patch('mlpstorage_py.main.logger') - def test_no_tempdir_warning_when_mlperf_results_dir_env_set( - self, mock_logger, mock_training_cls, monkeypatch - ): - """No tempdir warning when MLPERF_RESULTS_DIR is set in the environment. - - Even if results_dir happens to equal the DEFAULT_RESULTS_DIR constant that - was baked in at import time, the runtime env-var check prevents the warning. - """ - from mlpstorage_py.main import run_benchmark - from mlpstorage_py.config import DEFAULT_RESULTS_DIR - - # Set the env var at runtime — the warning condition checks os.environ live - monkeypatch.setenv('MLPERF_RESULTS_DIR', '/env/results') - mock_training_cls.return_value = _mock_benchmark() - - # Pass the old DEFAULT_RESULTS_DIR value; the env-var check still suppresses warning - run_benchmark(_make_args(DEFAULT_RESULTS_DIR), '20260427_120000') - - for call in mock_logger.warning.call_args_list: - text = str(call).lower() - assert 'temp directory' not in text, ( - f"Unexpected tempdir warning when MLPERF_RESULTS_DIR was set: {call}" - ) From 10b108d86899cb6236b8a3a799529825974da73a Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:49:24 -0700 Subject: [PATCH 15/45] test(06-03): add 6-run unet3d + 3-run resnet50 training fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 6-run unet3d fixture with anomalous 10x warmup at 20260701_100000 (start field matches 20260701_101500 to fire _process_single_run DLIO id-collision detection at report_generator.py:544-571) — anchors TEST-14 warmup exclusion assertion. - 3-run resnet50 partial fixture — triggers D-24 _INVALID_MSG_TRAINING_COUNT for D-27 rules-strict coverage. - Existing 20250115_143022 fixture byte-unchanged. --- .../resnet50/run/20260702_100000/summary.json | 30 +++++++++++++++ .../training_resnet50_metadata.json | 37 +++++++++++++++++++ .../resnet50/run/20260702_101500/summary.json | 30 +++++++++++++++ .../training_resnet50_metadata.json | 37 +++++++++++++++++++ .../resnet50/run/20260702_103000/summary.json | 30 +++++++++++++++ .../training_resnet50_metadata.json | 37 +++++++++++++++++++ .../unet3d/run/20260701_100000/summary.json | 32 ++++++++++++++++ .../training_unet3d_metadata.json | 37 +++++++++++++++++++ .../unet3d/run/20260701_101500/summary.json | 32 ++++++++++++++++ .../training_unet3d_metadata.json | 37 +++++++++++++++++++ .../unet3d/run/20260701_103000/summary.json | 32 ++++++++++++++++ .../training_unet3d_metadata.json | 37 +++++++++++++++++++ .../unet3d/run/20260701_104500/summary.json | 32 ++++++++++++++++ .../training_unet3d_metadata.json | 37 +++++++++++++++++++ .../unet3d/run/20260701_110000/summary.json | 32 ++++++++++++++++ .../training_unet3d_metadata.json | 37 +++++++++++++++++++ .../unet3d/run/20260701_111500/summary.json | 32 ++++++++++++++++ .../training_unet3d_metadata.json | 37 +++++++++++++++++++ 18 files changed, 615 insertions(+) create mode 100644 tests/fixtures/sample_results/training/resnet50/run/20260702_100000/summary.json create mode 100644 tests/fixtures/sample_results/training/resnet50/run/20260702_100000/training_resnet50_metadata.json create mode 100644 tests/fixtures/sample_results/training/resnet50/run/20260702_101500/summary.json create mode 100644 tests/fixtures/sample_results/training/resnet50/run/20260702_101500/training_resnet50_metadata.json create mode 100644 tests/fixtures/sample_results/training/resnet50/run/20260702_103000/summary.json create mode 100644 tests/fixtures/sample_results/training/resnet50/run/20260702_103000/training_resnet50_metadata.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_100000/summary.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_100000/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_101500/summary.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_101500/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_103000/summary.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_103000/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_104500/summary.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_104500/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_110000/summary.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_110000/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_111500/summary.json create mode 100644 tests/fixtures/sample_results/training/unet3d/run/20260701_111500/training_unet3d_metadata.json diff --git a/tests/fixtures/sample_results/training/resnet50/run/20260702_100000/summary.json b/tests/fixtures/sample_results/training/resnet50/run/20260702_100000/summary.json new file mode 100644 index 00000000..cfe18e96 --- /dev/null +++ b/tests/fixtures/sample_results/training/resnet50/run/20260702_100000/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-02 10:00:00", + "end": "2026-07-02 10:05:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "resnet50_h100", + "metric": { + "train_au_percentage": [ + 92.0, + 91.8, + 92.1 + ], + "train_throughput_samples_per_second": [ + 2100.0, + 2098.0, + 2102.0 + ], + "train_io_throughput_MB_per_second": [ + 3200.0, + 3180.0, + 3220.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/resnet50/run/20260702_100000/training_resnet50_metadata.json b/tests/fixtures/sample_results/training/resnet50/run/20260702_100000/training_resnet50_metadata.json new file mode 100644 index 00000000..8f5e0c9d --- /dev/null +++ b/tests/fixtures/sample_results/training/resnet50/run/20260702_100000/training_resnet50_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "resnet50", + "command": "run", + "run_datetime": "20260702_100000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/resnet50/run/20260702_100000", + "parameters": { + "model": { + "name": "resnet50" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/resnet50", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/resnet50/run/20260702_101500/summary.json b/tests/fixtures/sample_results/training/resnet50/run/20260702_101500/summary.json new file mode 100644 index 00000000..0d1dec06 --- /dev/null +++ b/tests/fixtures/sample_results/training/resnet50/run/20260702_101500/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-02 10:15:00", + "end": "2026-07-02 10:20:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "resnet50_h100", + "metric": { + "train_au_percentage": [ + 92.0, + 91.8, + 92.1 + ], + "train_throughput_samples_per_second": [ + 2100.0, + 2098.0, + 2102.0 + ], + "train_io_throughput_MB_per_second": [ + 3200.0, + 3180.0, + 3220.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/resnet50/run/20260702_101500/training_resnet50_metadata.json b/tests/fixtures/sample_results/training/resnet50/run/20260702_101500/training_resnet50_metadata.json new file mode 100644 index 00000000..9634152b --- /dev/null +++ b/tests/fixtures/sample_results/training/resnet50/run/20260702_101500/training_resnet50_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "resnet50", + "command": "run", + "run_datetime": "20260702_101500", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/resnet50/run/20260702_101500", + "parameters": { + "model": { + "name": "resnet50" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/resnet50", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/resnet50/run/20260702_103000/summary.json b/tests/fixtures/sample_results/training/resnet50/run/20260702_103000/summary.json new file mode 100644 index 00000000..aaa5d790 --- /dev/null +++ b/tests/fixtures/sample_results/training/resnet50/run/20260702_103000/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-02 10:30:00", + "end": "2026-07-02 10:35:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "resnet50_h100", + "metric": { + "train_au_percentage": [ + 92.0, + 91.8, + 92.1 + ], + "train_throughput_samples_per_second": [ + 2100.0, + 2098.0, + 2102.0 + ], + "train_io_throughput_MB_per_second": [ + 3200.0, + 3180.0, + 3220.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/resnet50/run/20260702_103000/training_resnet50_metadata.json b/tests/fixtures/sample_results/training/resnet50/run/20260702_103000/training_resnet50_metadata.json new file mode 100644 index 00000000..0d26b4dd --- /dev/null +++ b/tests/fixtures/sample_results/training/resnet50/run/20260702_103000/training_resnet50_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "resnet50", + "command": "run", + "run_datetime": "20260702_103000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/resnet50/run/20260702_103000", + "parameters": { + "model": { + "name": "resnet50" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/resnet50", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_100000/summary.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_100000/summary.json new file mode 100644 index 00000000..ad5b3d2e --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_100000/summary.json @@ -0,0 +1,32 @@ +{ + "start": "2026-07-01 10:15:00", + "end": "2026-07-01 10:20:00", + "num_accelerators": 8, + "num_hosts": 2, + "host_memory_GB": [ + 256, + 256 + ], + "host_cpu_count": [ + 64, + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [ + 10.0, + 10.5, + 10.2 + ], + "train_throughput_samples_per_second": [ + 12500.0, + 12480.0, + 12510.0 + ], + "train_io_throughput_MB_per_second": [ + 45000.0, + 44800.0, + 45100.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_100000/training_unet3d_metadata.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_100000/training_unet3d_metadata.json new file mode 100644 index 00000000..7054e90a --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_100000/training_unet3d_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260701_100000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260701_100000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_101500/summary.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_101500/summary.json new file mode 100644 index 00000000..6c677213 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_101500/summary.json @@ -0,0 +1,32 @@ +{ + "start": "2026-07-01 10:15:00", + "end": "2026-07-01 10:20:00", + "num_accelerators": 8, + "num_hosts": 2, + "host_memory_GB": [ + 256, + 256 + ], + "host_cpu_count": [ + 64, + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [ + 95.0, + 94.8, + 95.2 + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_101500/training_unet3d_metadata.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_101500/training_unet3d_metadata.json new file mode 100644 index 00000000..c7da57e7 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_101500/training_unet3d_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260701_101500", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260701_101500", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_103000/summary.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_103000/summary.json new file mode 100644 index 00000000..ca461ad7 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_103000/summary.json @@ -0,0 +1,32 @@ +{ + "start": "2026-07-01 10:30:00", + "end": "2026-07-01 10:35:00", + "num_accelerators": 8, + "num_hosts": 2, + "host_memory_GB": [ + 256, + 256 + ], + "host_cpu_count": [ + 64, + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [ + 95.1, + 94.9, + 95.0 + ], + "train_throughput_samples_per_second": [ + 1252.0, + 1249.0, + 1251.0 + ], + "train_io_throughput_MB_per_second": [ + 4510.0, + 4490.0, + 4500.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_103000/training_unet3d_metadata.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_103000/training_unet3d_metadata.json new file mode 100644 index 00000000..027bf183 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_103000/training_unet3d_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260701_103000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260701_103000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_104500/summary.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_104500/summary.json new file mode 100644 index 00000000..0cc9095b --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_104500/summary.json @@ -0,0 +1,32 @@ +{ + "start": "2026-07-01 10:45:00", + "end": "2026-07-01 10:50:00", + "num_accelerators": 8, + "num_hosts": 2, + "host_memory_GB": [ + 256, + 256 + ], + "host_cpu_count": [ + 64, + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [ + 94.9, + 95.0, + 95.1 + ], + "train_throughput_samples_per_second": [ + 1249.0, + 1250.0, + 1251.0 + ], + "train_io_throughput_MB_per_second": [ + 4495.0, + 4500.0, + 4505.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_104500/training_unet3d_metadata.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_104500/training_unet3d_metadata.json new file mode 100644 index 00000000..efc1b724 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_104500/training_unet3d_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260701_104500", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260701_104500", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_110000/summary.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_110000/summary.json new file mode 100644 index 00000000..125291e9 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_110000/summary.json @@ -0,0 +1,32 @@ +{ + "start": "2026-07-01 11:00:00", + "end": "2026-07-01 11:05:00", + "num_accelerators": 8, + "num_hosts": 2, + "host_memory_GB": [ + 256, + 256 + ], + "host_cpu_count": [ + 64, + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [ + 95.2, + 94.9, + 94.9 + ], + "train_throughput_samples_per_second": [ + 1251.0, + 1248.0, + 1249.0 + ], + "train_io_throughput_MB_per_second": [ + 4505.0, + 4485.0, + 4490.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_110000/training_unet3d_metadata.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_110000/training_unet3d_metadata.json new file mode 100644 index 00000000..c62ffa11 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_110000/training_unet3d_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260701_110000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260701_110000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_111500/summary.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_111500/summary.json new file mode 100644 index 00000000..8c502606 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_111500/summary.json @@ -0,0 +1,32 @@ +{ + "start": "2026-07-01 11:15:00", + "end": "2026-07-01 11:20:00", + "num_accelerators": 8, + "num_hosts": 2, + "host_memory_GB": [ + 256, + 256 + ], + "host_cpu_count": [ + 64, + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [ + 95.0, + 95.1, + 94.9 + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1252.0, + 1248.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4515.0, + 4485.0 + ] + } +} diff --git a/tests/fixtures/sample_results/training/unet3d/run/20260701_111500/training_unet3d_metadata.json b/tests/fixtures/sample_results/training/unet3d/run/20260701_111500/training_unet3d_metadata.json new file mode 100644 index 00000000..f2b12ec2 --- /dev/null +++ b/tests/fixtures/sample_results/training/unet3d/run/20260701_111500/training_unet3d_metadata.json @@ -0,0 +1,37 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260701_111500", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260701_111500", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": { + "dataset.num_files_train": "42000" + }, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} From 1a78c0ab7bbd1942521c18a396ab8a94950acf0b Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:49:52 -0700 Subject: [PATCH 16/45] test(06-03): add 10-op happy + 7-op partial checkpointing fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 10-op happy path (20260703_100000) — D-20 canonical shape: read and write throughput lists each have EXACTLY 10 elements. - 7-op partial (20260703_120000) — TEST-15 D-24 _INVALID_MSG_CHECKPOINT_COUNT trigger. - Existing 20250115_150000 fixture untouched. --- .../checkpointing_llama3-8b_metadata.json | 24 ++++++++++++ .../run/20260703_100000/summary.json | 39 +++++++++++++++++++ .../checkpointing_llama3-8b_metadata.json | 24 ++++++++++++ .../run/20260703_120000/summary.json | 33 ++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/checkpointing_llama3-8b_metadata.json create mode 100644 tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/summary.json create mode 100644 tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/checkpointing_llama3-8b_metadata.json create mode 100644 tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/summary.json diff --git a/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/checkpointing_llama3-8b_metadata.json b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/checkpointing_llama3-8b_metadata.json new file mode 100644 index 00000000..9cab6a6d --- /dev/null +++ b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/checkpointing_llama3-8b_metadata.json @@ -0,0 +1,24 @@ +{ + "benchmark_type": "checkpointing", + "model": "llama3-8b", + "command": "run", + "run_datetime": "20260703_100000", + "num_processes": 8, + "parameters": { + "model": { + "name": "llama3_8b" + }, + "checkpoint": { + "checkpoint_folder": "/data/checkpoints", + "num_checkpoints_read": 1, + "num_checkpoints_write": 1, + "mode": "default" + }, + "workflow": { + "generate_data": false, + "train": false, + "checkpoint": true + } + }, + "override_parameters": {} +} diff --git a/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/summary.json b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/summary.json new file mode 100644 index 00000000..9ff78a1b --- /dev/null +++ b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_100000/summary.json @@ -0,0 +1,39 @@ +{ + "start": "2026-07-03 10:00:00", + "end": "2026-07-03 10:08:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 512 + ], + "host_cpu_count": [ + 64 + ], + "workload": "llama3_8b", + "metric": { + "checkpoint_read_throughput_GB_per_second": [ + 52.1, + 51.8, + 52.0, + 51.9, + 52.2, + 52.0, + 51.7, + 52.1, + 51.9, + 52.0 + ], + "checkpoint_write_throughput_GB_per_second": [ + 45.2, + 44.8, + 45.0, + 45.1, + 44.9, + 45.0, + 45.3, + 44.8, + 45.0, + 45.1 + ] + } +} diff --git a/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/checkpointing_llama3-8b_metadata.json b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/checkpointing_llama3-8b_metadata.json new file mode 100644 index 00000000..a6b2a793 --- /dev/null +++ b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/checkpointing_llama3-8b_metadata.json @@ -0,0 +1,24 @@ +{ + "benchmark_type": "checkpointing", + "model": "llama3-8b", + "command": "run", + "run_datetime": "20260703_120000", + "num_processes": 8, + "parameters": { + "model": { + "name": "llama3_8b" + }, + "checkpoint": { + "checkpoint_folder": "/data/checkpoints", + "num_checkpoints_read": 1, + "num_checkpoints_write": 1, + "mode": "default" + }, + "workflow": { + "generate_data": false, + "train": false, + "checkpoint": true + } + }, + "override_parameters": {} +} diff --git a/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/summary.json b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/summary.json new file mode 100644 index 00000000..3a394f8d --- /dev/null +++ b/tests/fixtures/sample_results/checkpointing/llama3-8b/run/20260703_120000/summary.json @@ -0,0 +1,33 @@ +{ + "start": "2026-07-03 12:00:00", + "end": "2026-07-03 12:05:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 512 + ], + "host_cpu_count": [ + 64 + ], + "workload": "llama3_8b", + "metric": { + "checkpoint_read_throughput_GB_per_second": [ + 52.0, + 51.9, + 52.1, + 51.8, + 52.0, + 51.9, + 52.0 + ], + "checkpoint_write_throughput_GB_per_second": [ + 45.0, + 44.9, + 45.1, + 44.8, + 45.0, + 44.9, + 45.0 + ] + } +} From e0d535b9b4c32e64e72abf804563f7257aea0877 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:50:16 -0700 Subject: [PATCH 17/45] test(06-03): add vdb in-summary-recall + recall_stats fallback fixtures - milvus/hnsw/run/20260704_100000: recall present in summary.json (D-21 primary in-summary path). - pgvector/ivfflat/run/20260704_120000: summary.json omits recall, sibling recall_stats.json present (D-21 fallback path per vdb_checks.py:462-469). recall_stats shape mirrors calc_recall output from vdb_benchmark/vdbbench/simple_bench.py:140. - Both summaries carry all five _REQUIRED_METRIC_FIELDS. --- .../milvus/hnsw/run/20260704_100000/summary.json | 13 +++++++++++++ .../ivfflat/run/20260704_120000/recall_stats.json | 14 ++++++++++++++ .../ivfflat/run/20260704_120000/summary.json | 12 ++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 tests/fixtures/sample_results/vdb/milvus/hnsw/run/20260704_100000/summary.json create mode 100644 tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json create mode 100644 tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/summary.json diff --git a/tests/fixtures/sample_results/vdb/milvus/hnsw/run/20260704_100000/summary.json b/tests/fixtures/sample_results/vdb/milvus/hnsw/run/20260704_100000/summary.json new file mode 100644 index 00000000..6cac2335 --- /dev/null +++ b/tests/fixtures/sample_results/vdb/milvus/hnsw/run/20260704_100000/summary.json @@ -0,0 +1,13 @@ +{ + "throughput_qps": 1250.5, + "mean_latency_ms": 12.3, + "p95_latency_ms": 18.7, + "p99_latency_ms": 25.4, + "p999_latency_ms": 41.2, + "recall": 0.982, + "engine": "milvus", + "index_type": "hnsw", + "workload": "milvus_hnsw", + "start": "2026-07-04 10:00:00", + "end": "2026-07-04 10:15:00" +} diff --git a/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json b/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json new file mode 100644 index 00000000..25e7ae15 --- /dev/null +++ b/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json @@ -0,0 +1,14 @@ +{ + "recall_at_k": 0.975, + "num_queries_evaluated": 10000, + "k": 10, + "min_recall": 0.85, + "max_recall": 1.0, + "mean_recall": 0.975, + "median_recall": 0.98, + "p5_recall": 0.9, + "p95_recall": 1.0, + "p99_recall": 1.0, + "per_query_recall": [], + "recall_by_query": {} +} diff --git a/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/summary.json b/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/summary.json new file mode 100644 index 00000000..386d2854 --- /dev/null +++ b/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/summary.json @@ -0,0 +1,12 @@ +{ + "throughput_qps": 980.2, + "mean_latency_ms": 15.6, + "p95_latency_ms": 22.1, + "p99_latency_ms": 30.4, + "p999_latency_ms": 48.7, + "engine": "pgvector", + "index_type": "ivfflat", + "workload": "pgvector_ivfflat", + "start": "2026-07-04 12:00:00", + "end": "2026-07-04 12:15:00" +} From c2e429a1014121b522a3ee3d9ee76bb1609c0963 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:50:43 -0700 Subject: [PATCH 18/45] test(06-03): add kvcache multi-option fixture (D-16 flattening, D-22 pass-through) - kvcache/llama3-8b/run/20260704_140000 with top-level aggregated_* fields and options dict containing profile_a and profile_b sub-dicts (each mirroring kvcache._aggregate_option_results:791-803 shape). - profile_a.aggregated_write_bandwidth_gbps = 0.0 anchors the D-22 pass-through boundary test: reportgen must emit 0.0 verbatim. --- .../kvcache_llama3-8b_metadata.json | 21 +++++++++ .../run/20260704_140000/summary.json | 43 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/kvcache_llama3-8b_metadata.json create mode 100644 tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/summary.json diff --git a/tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/kvcache_llama3-8b_metadata.json b/tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/kvcache_llama3-8b_metadata.json new file mode 100644 index 00000000..6ecfe1de --- /dev/null +++ b/tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/kvcache_llama3-8b_metadata.json @@ -0,0 +1,21 @@ +{ + "benchmark_type": "kv_cache", + "model": "llama3-8b", + "command": "run", + "run_datetime": "20260704_140000", + "num_processes": 8, + "parameters": { + "model": { + "name": "llama3-8b" + }, + "kvcache": { + "performance_profile": "balanced", + "trials": 3 + }, + "workflow": { + "generate_data": false, + "train": false + } + }, + "override_parameters": {} +} diff --git a/tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/summary.json b/tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/summary.json new file mode 100644 index 00000000..07db2e38 --- /dev/null +++ b/tests/fixtures/sample_results/kvcache/llama3-8b/run/20260704_140000/summary.json @@ -0,0 +1,43 @@ +{ + "schema_version": "1.0", + "run_datetime": "20260704_140000", + "npernode": 8, + "host_count": 1, + "total_ranks": 8, + "trials": 3, + "aggregated_read_bandwidth_gbps": 45.2, + "aggregated_write_bandwidth_gbps": 38.1, + "aggregated_avg_throughput_tokens_per_sec": 12500.0, + "aggregated_storage_throughput_tokens_per_sec": 12300.0, + "aggregated_p95_latency_ms": 18.4, + "performance_profile": "balanced", + "model": "llama3-8b", + "options": { + "profile_a": { + "option": "profile_a", + "aggregated_read_bandwidth_gbps": 42.0, + "aggregated_write_bandwidth_gbps": 0.0, + "aggregated_avg_throughput_tokens_per_sec": 11000.0, + "aggregated_storage_throughput_tokens_per_sec": 10900.0, + "aggregated_p95_latency_ms": 20.1, + "rank_count": 8, + "trial_count": 3, + "partial_failure": false, + "missing_files": [], + "cpu_tier_ranks": [] + }, + "profile_b": { + "option": "profile_b", + "aggregated_read_bandwidth_gbps": 48.5, + "aggregated_write_bandwidth_gbps": 41.2, + "aggregated_avg_throughput_tokens_per_sec": 14000.0, + "aggregated_storage_throughput_tokens_per_sec": 13700.0, + "aggregated_p95_latency_ms": 16.8, + "rank_count": 8, + "trial_count": 3, + "partial_failure": false, + "missing_files": [], + "cpu_tier_ranks": [] + } + } +} From ca2b7c1bf1a487c0d971761ec947009f9fbc3190 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:51:09 -0700 Subject: [PATCH 19/45] test(06-03): add whatif training fixture (D-29 skip-rules-strict anchor) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - whatif/training/unet3d/run/ with 3 partial training runs (would trigger D-24 _INVALID_MSG_TRAINING_COUNT in a closed/open context; per D-29 the whatif category value causes reportgen to SKIP rules-strict — the row must emit with category='whatif', not category='INVALID'). - metric lists are non-empty because aggregation math still runs on projected metrics per D-29. --- .../unet3d/run/20260705_100000/summary.json | 30 ++++++++++++++++ .../training_unet3d_metadata.json | 35 +++++++++++++++++++ .../unet3d/run/20260705_101500/summary.json | 30 ++++++++++++++++ .../training_unet3d_metadata.json | 35 +++++++++++++++++++ .../unet3d/run/20260705_103000/summary.json | 30 ++++++++++++++++ .../training_unet3d_metadata.json | 35 +++++++++++++++++++ 6 files changed, 195 insertions(+) create mode 100644 tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/summary.json create mode 100644 tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/summary.json create mode 100644 tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/summary.json create mode 100644 tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/training_unet3d_metadata.json diff --git a/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/summary.json b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/summary.json new file mode 100644 index 00000000..4290e480 --- /dev/null +++ b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-05 10:00:00", + "end": "2026-07-05 10:05:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "unet3d_h100_whatif", + "metric": { + "train_au_percentage": [ + 95.0, + 94.8, + 95.1 + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/training_unet3d_metadata.json b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/training_unet3d_metadata.json new file mode 100644 index 00000000..096d3c6d --- /dev/null +++ b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_100000/training_unet3d_metadata.json @@ -0,0 +1,35 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260705_100000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/whatif/training/unet3d/run/20260705_100000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": {}, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/summary.json b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/summary.json new file mode 100644 index 00000000..d2eef89c --- /dev/null +++ b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-05 10:15:00", + "end": "2026-07-05 10:20:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "unet3d_h100_whatif", + "metric": { + "train_au_percentage": [ + 95.0, + 94.8, + 95.1 + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/training_unet3d_metadata.json b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/training_unet3d_metadata.json new file mode 100644 index 00000000..da954b14 --- /dev/null +++ b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_101500/training_unet3d_metadata.json @@ -0,0 +1,35 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260705_101500", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/whatif/training/unet3d/run/20260705_101500", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": {}, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/summary.json b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/summary.json new file mode 100644 index 00000000..ce118780 --- /dev/null +++ b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-05 10:30:00", + "end": "2026-07-05 10:35:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "unet3d_h100_whatif", + "metric": { + "train_au_percentage": [ + 95.0, + 94.8, + 95.1 + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/training_unet3d_metadata.json b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/training_unet3d_metadata.json new file mode 100644 index 00000000..1448660b --- /dev/null +++ b/tests/fixtures/sample_results/whatif/training/unet3d/run/20260705_103000/training_unet3d_metadata.json @@ -0,0 +1,35 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260705_103000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/whatif/training/unet3d/run/20260705_103000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": {}, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} From f39a523ad1c41291490afbf92ad8d0f46cb53dc9 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:51:58 -0700 Subject: [PATCH 20/45] test(06-03): add multi-orgname submission fixture (D-08 collection anchor) - multi_orgname/closed/{acme,beta_corp}/results/system-{a,b}/training/unet3d/run/ replicates the canonical CLOSED layout for two distinct orgnames side-by-side. Used by 06-05/06-06 to verify D-08: one top-level results.{csv,json} contains rows from BOTH orgs distinguished by the orgname column. Note: canonical layout includes a 'results/' path segment that matches the repo .gitignore's 'results/' output-dir pattern. Files force-added since the fixture must mirror the production tree shape that generate_output_location emits (Rule 3 blocking-issue fix: gitignore rule is intended for benchmark runtime output, not test fixtures under tests/fixtures/). --- .../unet3d/run/20260706_100000/summary.json | 30 ++++++++++++++++ .../training_unet3d_metadata.json | 35 +++++++++++++++++++ .../unet3d/run/20260706_101500/summary.json | 30 ++++++++++++++++ .../training_unet3d_metadata.json | 35 +++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/summary.json create mode 100644 tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/summary.json create mode 100644 tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/training_unet3d_metadata.json diff --git a/tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/summary.json b/tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/summary.json new file mode 100644 index 00000000..73de41d8 --- /dev/null +++ b/tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-06 10:00:00", + "end": "2026-07-06 10:05:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "unet3d_h100_acme", + "metric": { + "train_au_percentage": [ + 95.0, + 94.8, + 95.1 + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/training_unet3d_metadata.json b/tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/training_unet3d_metadata.json new file mode 100644 index 00000000..46a6dc27 --- /dev/null +++ b/tests/fixtures/sample_results/multi_orgname/closed/acme/results/system-a/training/unet3d/run/20260706_100000/training_unet3d_metadata.json @@ -0,0 +1,35 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260706_100000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/closed/acme/results/system-a/training/unet3d/run/20260706_100000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": {}, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/summary.json b/tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/summary.json new file mode 100644 index 00000000..9b1c1b30 --- /dev/null +++ b/tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/summary.json @@ -0,0 +1,30 @@ +{ + "start": "2026-07-06 10:15:00", + "end": "2026-07-06 10:15:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "unet3d_h100_beta_corp", + "metric": { + "train_au_percentage": [ + 95.0, + 94.8, + 95.1 + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/training_unet3d_metadata.json b/tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/training_unet3d_metadata.json new file mode 100644 index 00000000..e44d0081 --- /dev/null +++ b/tests/fixtures/sample_results/multi_orgname/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500/training_unet3d_metadata.json @@ -0,0 +1,35 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260706_101500", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/closed/beta_corp/results/system-b/training/unet3d/run/20260706_101500", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": {}, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} From bf35393f14ca6c8bd195adbca795f38f7050b260 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:52:15 -0700 Subject: [PATCH 21/45] test(06-03): add empty-model-dir fixture (D-03 header-only anchor) - empty_model/training/unet3d/run/ has a valid model+run directory layout but ZERO timestamp subdirectories. Used by 06-06 to verify D-03: reportgen must regenerate results.csv (header row only) and results.json ([]) for empty model dirs, not skip them. - .gitkeep preserves the empty directory under git. --- .../sample_results/empty_model/training/unet3d/run/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/fixtures/sample_results/empty_model/training/unet3d/run/.gitkeep diff --git a/tests/fixtures/sample_results/empty_model/training/unet3d/run/.gitkeep b/tests/fixtures/sample_results/empty_model/training/unet3d/run/.gitkeep new file mode 100644 index 00000000..e69de29b From 0772c4b79bbb6924115846198fa5876c381b4025 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:52:45 -0700 Subject: [PATCH 22/45] test(06-03): add degenerate fixtures (D-23 empty, NaN, missing summary) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - degenerate/empty_metric/training/unet3d/run/20260707_100000: train_au_percentage=[] triggers D-23 empty-list INVALID downgrade; other metrics stay valid to anchor 'per-metric null / row category INVALID' behavior. - degenerate/nan_metric/training/unet3d/run/20260707_120000: train_au_percentage=[null] documents helper's behavior on NaN (JSON has no NaN literal; null stands in per D-23 discussion). - degenerate/missing_summary/training/unet3d/run/20260707_140000/: no summary.json, just .gitkeep — covers T-06-06 FileNotFoundError graceful-degradation path. --- .../unet3d/run/20260707_100000/summary.json | 26 ++++++++++++++ .../training_unet3d_metadata.json | 35 +++++++++++++++++++ .../unet3d/run/20260707_140000/.gitkeep | 0 .../unet3d/run/20260707_120000/summary.json | 28 +++++++++++++++ .../training_unet3d_metadata.json | 35 +++++++++++++++++++ 5 files changed, 124 insertions(+) create mode 100644 tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/summary.json create mode 100644 tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/training_unet3d_metadata.json create mode 100644 tests/fixtures/sample_results/degenerate/missing_summary/training/unet3d/run/20260707_140000/.gitkeep create mode 100644 tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/summary.json create mode 100644 tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/training_unet3d_metadata.json diff --git a/tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/summary.json b/tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/summary.json new file mode 100644 index 00000000..d37e00ce --- /dev/null +++ b/tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/summary.json @@ -0,0 +1,26 @@ +{ + "start": "2026-07-07 10:00:00", + "end": "2026-07-07 10:05:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/training_unet3d_metadata.json b/tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/training_unet3d_metadata.json new file mode 100644 index 00000000..16ef46aa --- /dev/null +++ b/tests/fixtures/sample_results/degenerate/empty_metric/training/unet3d/run/20260707_100000/training_unet3d_metadata.json @@ -0,0 +1,35 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260707_100000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260707_100000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": {}, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} diff --git a/tests/fixtures/sample_results/degenerate/missing_summary/training/unet3d/run/20260707_140000/.gitkeep b/tests/fixtures/sample_results/degenerate/missing_summary/training/unet3d/run/20260707_140000/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/summary.json b/tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/summary.json new file mode 100644 index 00000000..de083db7 --- /dev/null +++ b/tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/summary.json @@ -0,0 +1,28 @@ +{ + "start": "2026-07-07 12:00:00", + "end": "2026-07-07 12:05:00", + "num_accelerators": 8, + "num_hosts": 1, + "host_memory_GB": [ + 256 + ], + "host_cpu_count": [ + 64 + ], + "workload": "unet3d_h100", + "metric": { + "train_au_percentage": [ + null + ], + "train_throughput_samples_per_second": [ + 1250.0, + 1248.0, + 1252.0 + ], + "train_io_throughput_MB_per_second": [ + 4500.0, + 4480.0, + 4520.0 + ] + } +} diff --git a/tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/training_unet3d_metadata.json b/tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/training_unet3d_metadata.json new file mode 100644 index 00000000..b5ec131d --- /dev/null +++ b/tests/fixtures/sample_results/degenerate/nan_metric/training/unet3d/run/20260707_120000/training_unet3d_metadata.json @@ -0,0 +1,35 @@ +{ + "benchmark_type": "training", + "model": "unet3d", + "command": "run", + "run_datetime": "20260707_120000", + "num_processes": 8, + "accelerator": "h100", + "result_dir": "/tmp/results/training/unet3d/run/20260707_120000", + "parameters": { + "model": { + "name": "unet3d" + }, + "dataset": { + "num_files_train": 42000, + "num_subfolders_train": 0, + "data_folder": "/data/unet3d", + "format": "npz" + }, + "reader": { + "read_threads": 8, + "computation_threads": 1, + "prefetch_size": 2 + }, + "workflow": { + "generate_data": false, + "train": true, + "checkpoint": true + } + }, + "override_parameters": {}, + "system_info": { + "total_memory_bytes": 549755813888, + "total_cores": 128 + } +} From 34adfeb799e33704ae2ec4f414e7509c24e49dac Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:49:16 -0700 Subject: [PATCH 23/45] feat(06-02): add fmean import and D-24 verbatim INVALID message constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Import statistics.fmean + StatisticsError (STACK.md A-01 ADR — no numpy/pandas/scipy) - Import typing.Set for the forthcoming _aggregate_workload_metrics(runs, warmup_set) signature - Four module-scope D-24 verbatim message templates with {n}/{key}/{basename} placeholders: _INVALID_MSG_TRAINING_COUNT, _INVALID_MSG_WARMUP_UNDETECTED, _INVALID_MSG_CHECKPOINT_COUNT, _INVALID_MSG_EMPTY_METRIC - Leading DO NOT paraphrase comment (Phase 5 D-02 precedent); § character preserved Prep for Task 2 which introduces _aggregate_workload_metrics helper. Wiring into _process_workload_groups deferred to plan 06-04 per phase boundary. --- mlpstorage_py/report_generator.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/mlpstorage_py/report_generator.py b/mlpstorage_py/report_generator.py index fd339e7e..e141e0cc 100755 --- a/mlpstorage_py/report_generator.py +++ b/mlpstorage_py/report_generator.py @@ -15,7 +15,8 @@ import sys from dataclasses import dataclass -from typing import List, Dict, Any, Optional +from statistics import fmean, StatisticsError +from typing import List, Dict, Any, Optional, Set from mlpstorage_py.mlps_logging import setup_logging, apply_logging_options from mlpstorage_py.config import MLPS_DEBUG, BENCHMARK_TYPES, EXIT_CODE, PARAM_VALIDATION, LLM_MODELS, MODELS, ACCELERATORS @@ -30,6 +31,25 @@ ) +# D-24 verbatim INVALID message templates. DO NOT paraphrase — tests/unit/test_aggregation.py +# asserts these substrings verbatim (Phase 5 D-02 precedent). Formatted at call sites via +# ``.format(n=..., key=..., basename=...)``; the ``§`` (U+00A7) character is intentional and +# grep-tested. +_INVALID_MSG_TRAINING_COUNT = ( + "expected 6 training invocations per Rules.md §2.1.17 (1 warmup + 5 real); found {n}" +) +_INVALID_MSG_WARMUP_UNDETECTED = ( + "expected exactly 1 warmup invocation to be detected; found 0 in a 6-invocation set. " + "Cannot compute Rules.md §2.1.17 5-run mean." +) +_INVALID_MSG_CHECKPOINT_COUNT = ( + "expected 10 checkpoint operations per Rules.md §2.1.23; found {n}" +) +_INVALID_MSG_EMPTY_METRIC = ( + "metric {key} is empty in invocation {basename}; cannot aggregate" +) + + @dataclass class Result: """Container for a single benchmark run result.""" From ec1d26fc237a2268a341cbfcc1dfe68d6fd8ab57 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 16:53:40 -0700 Subject: [PATCH 24/45] feat(06-02): add _aggregate_workload_metrics dispatch + four per-type helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the pure aggregation helper on ReportGenerator, placed immediately before _process_workload_groups (the caller in 06-04): - _aggregate_workload_metrics(runs, warmup_set): dispatches on runs[0].benchmark_type across four private helper methods (D-19..D-22). - _aggregate_training (D-19): filters warmup via os.path.abspath(), intra-invocation fmean of each metric list, then inter-invocation fmean of the per-run means. Output key 'train_mean_of_' strips redundant 'train_' prefix (D-13). - _aggregate_checkpointing (D-20/D-28): warmup_set ignored per docstring; intra-list fmean per invocation; inter-invocation fmean only when len(runs) > 1. Output key 'checkpoint_mean_of_'. - _aggregate_vdb (D-21): pass-through of throughput/latency/recall fields from summary.json into vdb_* columns; recall_stats.json fallback per submission_checker/checks/vdb_checks.py:462-469; identity columns vdb_engine and vdb_index_type from BenchmarkRun.parameters. - _aggregate_kvcache (D-22/D-16/D-17): pass-through of top-level aggregated_* fields into kvcache_aggregated_* columns; per-option flattening into kvcache_option__ columns preserving the source 'aggregated_' word verbatim (D-17 grep-chain). - _load_workload_summary helper: reads /summary.json with OSError/ValueError logged and empty dict returned (mitigates T-06-06 without aborting reportgen). Empty metric lists raise statistics.StatisticsError with the metric key and invocation basename included (D-23) — the caller in 06-04 catches it and downgrades the workload to INVALID using the D-24 _INVALID_MSG_EMPTY_METRIC template. NO 'else 0.0' fallback, NO broad except swallowing StatisticsError. _process_workload_groups and the writer signatures are byte-unchanged (SC-6 preserved). Full unit suite passes with zero regressions. --- mlpstorage_py/report_generator.py | 367 ++++++++++++++++++++++++++++++ 1 file changed, 367 insertions(+) diff --git a/mlpstorage_py/report_generator.py b/mlpstorage_py/report_generator.py index e141e0cc..7f14d0b2 100755 --- a/mlpstorage_py/report_generator.py +++ b/mlpstorage_py/report_generator.py @@ -601,6 +601,373 @@ def _process_single_run(self, benchmark_run: BenchmarkRun) -> None: f"Run {benchmark_run.run_id} validated as {category.value.upper()}" ) + # ------------------------------------------------------------------ + # Per-workload aggregation helpers (D-19..D-22) + # ------------------------------------------------------------------ + # + # ``_aggregate_workload_metrics`` is the single point where per-workload + # aggregation math lives. It is added here (Plan 06-02) but not yet + # wired into ``_process_workload_groups``; wiring happens in Plan 06-04 + # (the TODO at ``metrics={}`` in this file's ``_process_workload_groups`` + # is the eventual call site). + # + # Dispatch on ``runs[0].benchmark_type``: + # - training -> ``_aggregate_training`` (D-19, 5-run mean) + # - checkpointing -> ``_aggregate_checkpointing`` (D-20/D-28) + # - vector_database-> ``_aggregate_vdb`` (D-21, pass-through) + # - kv_cache -> ``_aggregate_kvcache`` (D-22/D-16, pass-through) + # + # Empty-metric handling: raises ``statistics.StatisticsError`` — the + # helper deliberately does NOT swallow it (D-23). The caller in 06-04 + # is responsible for the ``except StatisticsError`` split from the + # broad ``except Exception:`` and for downgrading ``category`` to + # ``PARAM_VALIDATION.INVALID`` with the D-24 verbatim message. + + def _aggregate_workload_metrics( + self, + runs: List[BenchmarkRun], + warmup_set: Set[str], + ) -> Dict[str, Any]: + """ + Compute aggregated metrics for one workload (per D-19..D-22). + + Dispatches on ``runs[0].benchmark_type`` and returns the aggregated + metric dict that eventually populates the ``metrics=`` slot on the + workload ``Result`` (see the TODO at ``_process_workload_groups``). + + Args: + runs: The workload's ``BenchmarkRun`` invocations. For training, + a 6-invocation set (1 warmup + 5 real) after + ``_process_single_run`` collision detection. For + checkpointing, 1–2 invocations per Rules.md §2.1.23. For + vdb / kvcache, a single-invocation list. + warmup_set: Set of ABSOLUTE paths (as populated in + ``self.warmup_result_dirs``). Used ONLY by the training + branch to filter warmup runs out of the 5-run mean + (D-19). The checkpointing / vdb / kvcache branches ignore + this argument (D-20/D-28: no warmup for those types). + + Returns: + A dict of aggregated metric names to values. Key convention: + - training : ``train_mean_of_`` (D-13) + - checkpointing : ``checkpoint_mean_of_`` (D-13) + - vdb : ``vdb_`` pass-through (D-15) + - kvcache : ``kvcache_aggregated_<...>`` + + ``kvcache_option__aggregated_<...>`` (D-15..D-17) + + Raises: + statistics.StatisticsError: When a metric list is empty (training + or checkpointing branch). The caller in 06-04 catches this + and downgrades the workload's ``category`` to + ``PARAM_VALIDATION.INVALID`` per D-23. Do NOT swallow this + here — the split from the broad ``except Exception:`` is + the loud-failure contract (D-23; PITFALLS #3). + """ + if not runs: + return {} + bt = runs[0].benchmark_type + if bt == BENCHMARK_TYPES.training: + return self._aggregate_training(runs, warmup_set) + elif bt == BENCHMARK_TYPES.checkpointing: + return self._aggregate_checkpointing(runs) + elif bt == BENCHMARK_TYPES.vector_database: + return self._aggregate_vdb(runs) + elif bt == BENCHMARK_TYPES.kv_cache: + return self._aggregate_kvcache(runs) + else: + return {} + + def _aggregate_training( + self, + runs: List[BenchmarkRun], + warmup_set: Set[str], + ) -> Dict[str, float]: + """ + Training-branch aggregation (D-19, Rules.md §2.1.17). + + Filters ``runs`` down to the non-warmup invocations + (``abspath(run.result_dir) not in warmup_set``), then for each + metric key present in EVERY non-warmup invocation, computes: + + per_run = [fmean(inv.metrics[key]) for inv in non_warmup] + outer = fmean(per_run) + + emitting under ``train_mean_of_`` where basename is the + source key with any redundant ``train_`` prefix stripped (D-13). + + Empty inner metric list -> ``StatisticsError`` propagates (D-23). + Missing key in a given invocation is skipped via key intersection + (a partial-coverage key is not aggregated — the caller sees it + absent from the output). + + Count strictness (D-27) and warmup detection (D-26) are enforced + at the CALLER site in 06-04, not here — the helper focuses on + math. If ``non_warmup`` is empty after filtering, this raises + ``StatisticsError`` with a helpful ``args[0]`` so the caller can + surface INVALID. + """ + non_warmup = [ + r for r in runs + if os.path.abspath(r.result_dir or "") not in warmup_set + ] + if not non_warmup: + raise StatisticsError( + "no non-warmup invocations to aggregate for training workload" + ) + + # Intersection of metric keys across all non-warmup invocations. + # A key present in some invocations but not all is skipped — the + # caller sees it missing from the output and can decide. + metric_dicts = [(inv.metrics or {}) for inv in non_warmup] + common_keys = set(metric_dicts[0].keys()) + for md in metric_dicts[1:]: + common_keys &= set(md.keys()) + + out: Dict[str, float] = {} + for key in sorted(common_keys): + per_run_means: List[float] = [] + for inv in non_warmup: + metric_list = (inv.metrics or {}).get(key) + if not metric_list: + # Loud failure per D-23: empty metric list is NOT + # silently coerced to 0.0. Caller wraps this in a + # try/except StatisticsError and emits the D-24 + # ``_INVALID_MSG_EMPTY_METRIC`` template. + raise StatisticsError( + f"metric {key} is empty in invocation " + f"{os.path.basename(inv.result_dir or '')}" + ) + per_run_means.append(fmean(metric_list)) + outer = fmean(per_run_means) + # D-13: strip redundant ``train_`` prefix from the source key + # so the emitted column looks like ``train_mean_of_``. + basename = key + if basename.startswith("train_"): + basename = basename[len("train_"):] + out[f"train_mean_of_{basename}"] = outer + return out + + def _aggregate_checkpointing( + self, + runs: List[BenchmarkRun], + ) -> Dict[str, float]: + """ + Checkpointing-branch aggregation (D-20/D-28, Rules.md §2.1.23). + + ``warmup_set`` is not accepted here — checkpointing has no warmup + per Rules.md §2.1.23; the dispatcher in + ``_aggregate_workload_metrics`` ignores ``warmup_set`` for this + branch and calls this method without it. + + For each metric key present in every invocation's ``metric`` dict, + computes ``fmean(run.metrics[key])`` intra-list over the 10-op + list per invocation. If ``len(runs) > 1`` (rare per Rules.md + §2.1.23 which permits 1–2 timestamp directories), takes the + inter-invocation ``fmean`` for shape consistency with the + training branch. Op-count strictness (D-24 template c) is a + caller-side gate in 06-04, not here. + + Empty metric list -> ``StatisticsError`` propagates (D-23). Do + NOT copy the "fmean(x) if x, coerce to zero otherwise" idiom + from ``benchmarks/kvcache.py:793`` — that is the PITFALLS #3 + anti-pattern and the loud-failure principle forbids it here. + """ + if not runs: + raise StatisticsError( + "no checkpointing invocations to aggregate" + ) + + metric_dicts = [(r.metrics or {}) for r in runs] + common_keys = set(metric_dicts[0].keys()) + for md in metric_dicts[1:]: + common_keys &= set(md.keys()) + + out: Dict[str, float] = {} + for key in sorted(common_keys): + per_run_means: List[float] = [] + for r in runs: + metric_list = (r.metrics or {}).get(key) + if not metric_list: + raise StatisticsError( + f"metric {key} is empty in invocation " + f"{os.path.basename(r.result_dir or '')}" + ) + per_run_means.append(fmean(metric_list)) + outer = fmean(per_run_means) if len(per_run_means) > 1 else per_run_means[0] + basename = key + if basename.startswith("checkpoint_"): + basename = basename[len("checkpoint_"):] + out[f"checkpoint_mean_of_{basename}"] = outer + return out + + def _aggregate_vdb( + self, + runs: List[BenchmarkRun], + ) -> Dict[str, Any]: + """ + VDB-branch aggregation (D-21) — pass-through, NOT math. + + vdb's internal ``vdb-aggregate`` tool (see + ``benchmarks/vectordbbench.py:508``) owns the math contract per + the D-22 boundary; this helper copies pre-computed values from + the workload's ``summary.json`` into ``vdb_*`` columns. + + Read fields (per ``submission_checker/checks/vdb_checks.py:44-51`` + ``_REQUIRED_METRIC_FIELDS``): ``throughput_qps``, + ``mean_latency_ms``, ``p95_latency_ms``, ``p99_latency_ms``, + ``p999_latency_ms``. Emit as ``vdb_`` (D-15). Missing + fields emit ``None`` (D-22 boundary: vdb owns validity — INVALID + rules-strict is training + checkpointing only). + + Recall fallback: if ``recall`` is absent from ``summary.json``, + fall back to ``/recall_stats.json`` per the pattern + at ``submission_checker/checks/vdb_checks.py:462-469``. + + Identity columns (D-15): ``vdb_engine`` and ``vdb_index_type`` + read from ``runs[0].parameters`` (the ``BenchmarkRun`` accessor + for CLI/YAML args on disk). + """ + run = runs[0] + summary = self._load_workload_summary(run) + + _VDB_METRIC_FIELDS = ( + "throughput_qps", + "mean_latency_ms", + "p95_latency_ms", + "p99_latency_ms", + "p999_latency_ms", + ) + out: Dict[str, Any] = {} + for field_name in _VDB_METRIC_FIELDS: + out[f"vdb_{field_name}"] = summary.get(field_name) + + # Recall (D-21) — first summary.json, then recall_stats.json fallback + # (per submission_checker/checks/vdb_checks.py:462-469). + recall = summary.get("recall") + if recall is None and run.result_dir: + recall_stats_path = os.path.join(run.result_dir, "recall_stats.json") + if os.path.isfile(recall_stats_path): + try: + with open(recall_stats_path, "r") as f: + recall_stats = json.load(f) + recall = recall_stats.get("recall") + except (OSError, ValueError) as e: + self.logger.warning( + f"vdb: could not read recall_stats.json at " + f"{recall_stats_path}: {e}" + ) + out["vdb_recall"] = recall + + # Identity columns (D-15). ``BenchmarkRun`` exposes CLI/YAML args + # via ``.parameters``; use ``.get`` so missing keys emit ``None``. + params = run.parameters or {} + out["vdb_engine"] = params.get("engine") + out["vdb_index_type"] = params.get("index_type") + return out + + def _aggregate_kvcache( + self, + runs: List[BenchmarkRun], + ) -> Dict[str, Any]: + """ + KVCache-branch aggregation (D-22/D-16/D-17) — pass-through, NOT math. + + kvcache's internal ``_aggregate_option_results`` at + ``benchmarks/kvcache.py:791-803`` owns the math contract per the + D-22 boundary. Trust the source verbatim — including any ``0.0`` + sentinel values from the "fmean-when-nonempty, coerce-to-zero + otherwise" idiom at ``benchmarks/kvcache.py:793``. That + silent-failure amplifier + belongs in kvcache, not in reportgen (PITFALLS #3). + + Emits: + - Top-level identity (D-15): ``kvcache_performance_profile`` + from ``runs[0].parameters['performance_profile']``. + - Top-level per-run aggregates (D-14) copied verbatim from + ``summary.json`` — ``kvcache_aggregated_read_bandwidth_gbps``, + ``kvcache_aggregated_write_bandwidth_gbps``, + ``kvcache_aggregated_avg_throughput_tokens_per_sec``, + ``kvcache_aggregated_storage_throughput_tokens_per_sec``, + ``kvcache_aggregated_p95_latency_ms``. + - Per-option flattening (D-16): for each + ``(option_name, option_dict)`` in ``summary['options']``, + emits ``kvcache_option__`` for each + ``(metric_name, value)`` pair. Since kvcache's internal keys + are ``aggregated_*`` (per ``benchmarks/kvcache.py:791-803``), + the resulting output columns look like + ``kvcache_option__aggregated_read_bandwidth_gbps`` — + preserving the grep-chain from source to output (D-17). + """ + run = runs[0] + summary = self._load_workload_summary(run) + + out: Dict[str, Any] = {} + # Identity column (D-15) — read from BenchmarkRun.parameters. + params = run.parameters or {} + out["kvcache_performance_profile"] = params.get("performance_profile") + + # Top-level per-run aggregates (D-14). Copy verbatim from source; + # Zero sentinels from kvcache's "fmean-or-zero" idiom are + # preserved (D-22 boundary — do NOT re-interpret). + _KVCACHE_TOPLEVEL_FIELDS = ( + "aggregated_read_bandwidth_gbps", + "aggregated_write_bandwidth_gbps", + "aggregated_avg_throughput_tokens_per_sec", + "aggregated_storage_throughput_tokens_per_sec", + "aggregated_p95_latency_ms", + ) + for field_name in _KVCACHE_TOPLEVEL_FIELDS: + out[f"kvcache_{field_name}"] = summary.get(field_name) + + # Per-option flattening (D-16). + options = summary.get("options") or {} + if not options: + self.logger.warning( + f"kvcache: summary.json at {run.result_dir!r} has no " + "'options' dict — per-option columns will be absent." + ) + else: + for option_name, option_dict in options.items(): + if not isinstance(option_dict, dict): + continue + for metric_name, value in option_dict.items(): + # D-17: keep the source ``aggregated_`` prefix + # verbatim to preserve the grep-chain from source + # summary.json field to output column name. + out[f"kvcache_option_{option_name}_{metric_name}"] = value + return out + + def _load_workload_summary(self, run: BenchmarkRun) -> Dict[str, Any]: + """ + Load the workload's ``summary.json`` for vdb / kvcache pass-through + branches. + + ``BenchmarkRun`` does not expose a ``summary_data`` accessor on + this branch of the code base, so the helper reads + ``/summary.json`` directly. Missing file or + malformed JSON logs a warning and returns ``{}`` — the caller + emits empty pass-through columns rather than aborting the whole + reportgen pass (threat T-06-06 mitigation). + """ + if not run.result_dir: + return {} + summary_path = os.path.join(run.result_dir, "summary.json") + if not os.path.isfile(summary_path): + self.logger.warning( + f"summary.json not found at {summary_path}; " + "pass-through columns will be empty." + ) + return {} + try: + with open(summary_path, "r") as f: + return json.load(f) + except (OSError, ValueError) as e: + self.logger.warning( + f"summary.json at {summary_path} could not be read: {e}; " + "pass-through columns will be empty." + ) + return {} + def _process_workload_groups(self, benchmark_runs: List[BenchmarkRun]) -> None: """ Group runs by workload and run submission-level validation. From 4f55fe9f133dc61ee0456d0ab2e82f9c7e839fe2 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 19:54:56 -0700 Subject: [PATCH 25/45] =?UTF-8?q?docs(06-01):=20add=20Rules.md=20=C2=A72.1?= =?UTF-8?q?.28=20aggregateResultsFile=20per=20D-30/D-34?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rules.md §2.1.28 defines the mechanical file shape produced by reportgen: bottom-up build from per-model files; one row per workload (no per-row discriminator column); fixed 6-column prefix (category, orgname, systemname, benchmark_type, model, accelerator); per-type-grouped columns with train_/ checkpoint_/vdb_/kvcache_ prefixes; alphabetical within each group; trailing issues column; category ∈ {closed, open, whatif, INVALID}; whatif skips rules-strict INVALID gates. Companion .planning/ROADMAP.md + .planning/REQUIREMENTS.md edits (SC-1/SC-5 rewrite; AGG-01..05 extension; Traceability + Coverage bump) are gitignored per repo policy and land alongside on-disk only. --- Rules.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Rules.md b/Rules.md index 1a69e844..f7aed0d5 100644 --- a/Rules.md +++ b/Rules.md @@ -337,6 +337,10 @@ root_folder (or any name you prefer) ├──system-name-2.yaml └──system-name-2.pdf ``` +2.1.28. **aggregateResultsFile** -- Reportgen assembles one top-level results file per invocation at `/results.{csv,json}` by walking each per-workload directory in the tree, computing that workload's aggregated row, writing the per-model `<...>//results.{csv,json}` files first, and then collecting those per-model rows into the top-level file (bottom-up build; the top-level file is a collection step, not an aggregation step). Each row in the top-level file represents ONE workload (submitter parlance: "run") of any benchmark type — training, checkpointing, vdb, or kvcache — with no per-invocation raw rows and no per-row discriminator column. Every reportgen invocation rebuilds these files from scratch, so deleted run directories disappear from the next report; empty model directories still emit a header-only CSV and an empty-list JSON. + +The row layout has a fixed 6-column prefix in this exact order: `category`, `orgname`, `systemname`, `benchmark_type`, `model`, `accelerator`. After the prefix, columns are grouped by benchmark type in the fixed order training → checkpointing → vdb → kvcache, with every column in a group carrying that group's prefix (`train_`, `checkpoint_`, `vdb_`, `kvcache_`). Within each group, columns are alphabetical. The trailing column is always `issues` — variable-length text carrying verbatim `Result.issues` messages joined by `; ` per §2.1.17 and §2.1.23 validation contracts. The `category` column takes one of four values: `closed`, `open`, `whatif`, or `INVALID`; the `INVALID` value is emitted when a workload violates the rules-strict counts described in §2.1.17 (training must be exactly 6 invocations, 1 warmup + 5 real) and §2.1.23 (checkpointing must have exactly 10 checkpoint operations per invocation). `whatif` rows are simulation output and skip the rules-strict INVALID gates entirely. + 2.29. **dlioLog** -- Since the "dlio_log" subdirectory has a similar structure in all cases, it is describe pictorially just below: ``` └── YYYYMMDD_HHmmss From 55b96dc7756054500381e051307b4e7c78a3bb69 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 20:01:29 -0700 Subject: [PATCH 26/45] feat(06-04): rewrite _process_workload_groups per D-05/D-23/D-26/D-27 - Refactor grouping key to per-benchmark-type dispatch (D-05): - training/checkpointing: (category, orgname, systemname, model, accelerator) - vdb: (category, orgname, systemname, engine, index_type) - kv_cache: (category, orgname, systemname, model, performance_profile) - Add _derive_workload_key + _derive_category/orgname/systemname_from_path helpers with defensive params handling (D-07). - Wire _aggregate_workload_metrics call at the former TODO site, wrapped in a targeted try/except statistics.StatisticsError (D-23 loud-failure). Inner catch handles the D-24 _INVALID_MSG_EMPTY_METRIC template with best-effort key/basename extraction; outer broad Exception catch preserved as pre-aggregation safety net (unchanged). - Insert rules-strict INVALID gates before aggregation dispatch: - D-27: training len(runs) != 6 -> _INVALID_MSG_TRAINING_COUNT - D-26: 6-invocation training with no warmup detected -> _INVALID_MSG_WARMUP_UNDETECTED - D-20/D-24: checkpointing metric list len != 10 -> _INVALID_MSG_CHECKPOINT_COUNT - Gates SKIP for category_str == 'whatif' per D-29. - Vdb + kvcache SKIP rules-strict entirely (D-22 pass-through). - Update _print_workload_details to read model/accelerator from benchmark_run[0] instead of unpacking the workload_key tuple (decouples printer from D-05 key shape evolution). - Add module-level 'import statistics' for the targeted except clause. - Update test_reporting.py::test_groups_by_workload to assert the new 5-tuple key shape (tail (model, accelerator) still stable). - Update test_reporting.py::test_prints_workload_results to use the 5-tuple key with empty (category, orgname, systemname) leading slots. --- mlpstorage_py/report_generator.py | 298 ++++++++++++++++++++++++++++-- tests/unit/test_reporting.py | 37 +++- 2 files changed, 312 insertions(+), 23 deletions(-) diff --git a/mlpstorage_py/report_generator.py b/mlpstorage_py/report_generator.py index 7f14d0b2..a2438b86 100755 --- a/mlpstorage_py/report_generator.py +++ b/mlpstorage_py/report_generator.py @@ -12,6 +12,7 @@ import json import os.path import pprint +import statistics import sys from dataclasses import dataclass @@ -968,36 +969,275 @@ def _load_workload_summary(self, run: BenchmarkRun) -> Dict[str, Any]: ) return {} + # ------------------------------------------------------------------ + # Workload identity derivation (D-05/D-07) + # ------------------------------------------------------------------ + _CATEGORY_PATH_TOKENS = ("closed", "open", "whatif") + + def _derive_category_from_path(self, benchmark_run: BenchmarkRun) -> Optional[str]: + """D-07 category derivation from the workload's on-disk path. + + Returns one of ``'closed'``, ``'open'``, ``'whatif'`` when the + workload's absolute result_dir contains that segment. Returns + ``None`` when no segment matches — the caller then falls back + to the upstream :class:`BenchmarkVerifier` category value. + """ + leaf = getattr(benchmark_run, 'result_dir', None) or "" + if not leaf: + return None + # Normalize to path segments; scan case-insensitively but return + # the lowercased canonical token so downstream string comparisons + # (D-29 ``category != 'whatif'``) are unambiguous. + parts = [p.lower() for p in os.path.abspath(leaf).split(os.sep) if p] + for token in self._CATEGORY_PATH_TOKENS: + if token in parts: + return token + return None + + def _derive_orgname_from_path(self, benchmark_run: BenchmarkRun) -> str: + """D-07 orgname derivation. + + Prefers ``benchmark_run.parameters['orgname']`` (when set at run + time); otherwise walks the workload's absolute result_dir looking + for the ``//results/`` triplet that the + canonical submission tree produces (see + ``_resolve_effective_results_dir``). Returns an empty string when + the path shape does not match — never raises. + """ + params = getattr(benchmark_run, 'parameters', None) + if isinstance(params, dict): + candidate = params.get('orgname') + if isinstance(candidate, str) and candidate: + return candidate + leaf = getattr(benchmark_run, 'result_dir', None) or "" + if not leaf: + return "" + parts = os.path.abspath(leaf).split(os.sep) + # Look for `//results/` where division is + # `closed`, `open`, or `whatif`. + for idx in range(len(parts) - 2): + if ( + parts[idx].lower() in self._CATEGORY_PATH_TOKENS + and parts[idx + 2] == "results" + ): + return parts[idx + 1] + return "" + + def _derive_systemname_from_path(self, benchmark_run: BenchmarkRun) -> str: + """D-07 systemname derivation. + + When ``self.args.systemname`` was supplied on the CLI, uses it. + Otherwise walks the workload's absolute path looking for + ``results//`` — the canonical submission tree shape. + Returns an empty string when the path shape does not match. + """ + systemname = None + if self.args is not None: + systemname = getattr(self.args, 'systemname', None) + if systemname: + return str(systemname) + leaf = getattr(benchmark_run, 'result_dir', None) or "" + if not leaf: + return "" + parts = os.path.abspath(leaf).split(os.sep) + for idx in range(len(parts) - 1): + if parts[idx] == "results": + return parts[idx + 1] + return "" + + def _derive_workload_key( + self, + benchmark_run: BenchmarkRun, + fallback_category: Optional[str] = None, + ) -> tuple: + """Return the per-benchmark-type grouping tuple (D-05/D-06). + + - training / checkpointing: ``(category, orgname, systemname, model, accelerator)`` + - vector_database: ``(category, orgname, systemname, engine, index_type)`` + - kv_cache: ``(category, orgname, systemname, model, performance_profile)`` + + ``category`` is derived from the workload's on-disk path segment + when possible (D-07); when the path does not match, falls back to + the caller-supplied ``fallback_category`` (typically the upstream + :class:`BenchmarkVerifier` category converted to string). + """ + category = self._derive_category_from_path(benchmark_run) + if category is None: + category = fallback_category or "" + orgname = self._derive_orgname_from_path(benchmark_run) + systemname = self._derive_systemname_from_path(benchmark_run) + bt = benchmark_run.benchmark_type + raw_params = getattr(benchmark_run, 'parameters', None) + params = raw_params if isinstance(raw_params, dict) else {} + + def _p(key: str) -> str: + """Return params[key] as a str only when it's a real string.""" + v = params.get(key) + return v if isinstance(v, str) else "" + + if bt == BENCHMARK_TYPES.vector_database: + return (category, orgname, systemname, _p('engine'), _p('index_type')) + if bt == BENCHMARK_TYPES.kv_cache: + model = str(benchmark_run.model or "") + return (category, orgname, systemname, model, _p('performance_profile')) + # training / checkpointing (and any other type) share the + # (category, orgname, systemname, model, accelerator) shape. + model = str(benchmark_run.model or "") + accelerator = str(benchmark_run.accelerator or "") + return (category, orgname, systemname, model, accelerator) + def _process_workload_groups(self, benchmark_runs: List[BenchmarkRun]) -> None: """ Group runs by workload and run submission-level validation. - Args: - benchmark_runs: List of all benchmark runs. + Grouping key is per-benchmark-type per D-05/D-06: + + - training / checkpointing: ``(category, orgname, systemname, model, accelerator)`` + - vector_database: ``(category, orgname, systemname, engine, index_type)`` + - kv_cache: ``(category, orgname, systemname, model, performance_profile)`` + + The aggregation dispatch (``_aggregate_workload_metrics``) sits + BELOW rules-strict INVALID gates (D-23/D-26/D-27) so that empty + metric lists never silently produce ``0.0`` and count/warmup + violations produce the D-24 verbatim message rather than a + computed-but-wrong value. ``whatif`` rows skip the rules-strict + gates entirely per D-29 (whatif is simulation, not submission). """ - # Group by (model, accelerator) for training, (model,) for others + # Group by per-benchmark-type key (D-05). Category comes from + # path derivation when possible; the upstream verifier still runs + # for each group to produce issues + a fallback category. workload_runs: Dict[tuple, List[BenchmarkRun]] = {} - for benchmark_run in benchmark_runs: - workload_key = (benchmark_run.model, benchmark_run.accelerator) - if workload_key not in workload_runs: - workload_runs[workload_key] = [] - workload_runs[workload_key].append(benchmark_run) + workload_key = self._derive_workload_key(benchmark_run) + workload_runs.setdefault(workload_key, []).append(benchmark_run) - # Run workload-level verifiers for workload_key, runs in workload_runs.items(): - model, accelerator = workload_key if not runs: continue try: + # workload_key = (category, orgname, systemname, id1, id2) + category_str, orgname_str, systemname_str, ident1, ident2 = workload_key self.logger.info( - f'Running submission verifiers for model: {model}, ' - f'accelerator: {accelerator} ({len(runs)} runs)' + f'Running submission verifiers for ' + f'{runs[0].benchmark_type.value if runs[0].benchmark_type else "?"} ' + f'({ident1}, {ident2}) — {len(runs)} runs' ) verifier = BenchmarkVerifier(*runs, logger=self.logger) - category = verifier.verify() - issues = verifier.issues + verifier_category = verifier.verify() + issues = list(verifier.issues) if verifier.issues else [] + + # If path-based category derivation returned "", fall back + # to the upstream verifier's category value. + if not category_str and verifier_category is not None: + try: + category_str = verifier_category.value + except AttributeError: + category_str = str(verifier_category) + + # ---------------------------------------------------------- + # Rules-strict INVALID gates (D-23/D-26/D-27) + # ---------------------------------------------------------- + # whatif rows SKIP these gates entirely per D-29 — whatif + # is a simulation, not a submission; INVALID semantics + # don't apply. + invalid_messages: List[str] = [] + if category_str != 'whatif': + bt = runs[0].benchmark_type + if bt == BENCHMARK_TYPES.training: + # D-27: training must be exactly 6 invocations + # (1 warmup + 5 real per Rules.md §2.1.17). + if len(runs) != 6: + invalid_messages.append( + _INVALID_MSG_TRAINING_COUNT.format(n=len(runs)) + ) + else: + # D-26: warmup MUST have been detected for a + # 6-invocation set. No lex-first-timestamp + # fallback — loud INVALID over silent guess. + warmup_present = any( + os.path.abspath(r.result_dir or "") in self.warmup_result_dirs + for r in runs + ) + if not warmup_present: + invalid_messages.append( + _INVALID_MSG_WARMUP_UNDETECTED + ) + elif bt == BENCHMARK_TYPES.checkpointing: + # D-20/D-24: each invocation's metric lists MUST + # have exactly 10 entries (Rules.md §2.1.23). + for run in runs: + violated = False + for _mkey, val in (run.metrics or {}).items(): + if isinstance(val, list) and len(val) != 10: + invalid_messages.append( + _INVALID_MSG_CHECKPOINT_COUNT.format(n=len(val)) + ) + violated = True + break # one violation per run is enough + if violated: + break + # vdb + kvcache SKIP rules-strict gates entirely per + # the D-22 pass-through boundary — their INVALID + # category (if any) is set by the upstream + # BenchmarkVerifier, not by Phase 6. + + # ---------------------------------------------------------- + # Aggregation dispatch — inner try/except for + # StatisticsError (D-23 loud-failure path). + # ---------------------------------------------------------- + aggregated_metrics: Dict[str, Any] = {} + if invalid_messages: + # Rules-strict already failed — skip aggregation + # entirely; the emitted row has no computed values. + category_str = 'INVALID' + aggregated_metrics = {} + else: + try: + aggregated_metrics = self._aggregate_workload_metrics(runs, self.warmup_result_dirs) # noqa: E501 + except statistics.StatisticsError as se: + # D-24 empty-metric template. Best-effort key / + # basename extraction from the exception message — + # the helper's raise-sites format the message as + # "metric is empty in invocation ". + category_str = 'INVALID' + msg_text = str(se) + parsed_key = "(unknown)" + parsed_basename = "(unknown)" + if msg_text.startswith("metric ") and " is empty in invocation " in msg_text: + try: + after_metric = msg_text[len("metric "):] + parsed_key, rest = after_metric.split( + " is empty in invocation ", 1 + ) + parsed_basename = rest.strip() + except ValueError: + pass + invalid_messages.append( + _INVALID_MSG_EMPTY_METRIC.format( + key=parsed_key, basename=parsed_basename + ) + ) + aggregated_metrics = {} + + # Merge rules-strict issues into the workload's Issue list. + # Preserve verbatim string (D-25 join happens at write + # time; Result.issues stays as a typed Issue list until + # then). + if invalid_messages: + for msg in invalid_messages: + issues.append(Issue(PARAM_VALIDATION.INVALID, msg)) + + # Resolve the Result.category. Path-derived 'whatif' and + # rules-strict 'INVALID' are strings; other paths use the + # verifier's PARAM_VALIDATION enum. Downstream writers + # coerce this to a string via ``.value`` when needed. + if category_str == 'INVALID': + result_category: Union[PARAM_VALIDATION, str] = PARAM_VALIDATION.INVALID + elif category_str == 'whatif': + result_category = 'whatif' + else: + result_category = verifier_category result = Result( multi=True, @@ -1006,8 +1246,8 @@ def _process_workload_groups(self, benchmark_runs: List[BenchmarkRun]) -> None: benchmark_command=runs[0].command, benchmark_model=runs[0].model, issues=issues, - category=category, - metrics={} # TODO: Add function to aggregate metrics + category=result_category, + metrics=aggregated_metrics, ) self.workload_results[workload_key] = result @@ -1137,10 +1377,32 @@ def _print_workload_details(self, workload_key: tuple, workload_result: Result) Print details for a workload submission. Args: - workload_key: Tuple of (model, accelerator). + workload_key: Per-benchmark-type workload identity tuple (D-05). + Shape depends on ``benchmark_type``: + - training/checkpointing: + ``(category, orgname, systemname, model, accelerator)`` + - vector_database: + ``(category, orgname, systemname, engine, index_type)`` + - kv_cache: + ``(category, orgname, systemname, model, performance_profile)`` + Model / accelerator identity is read from + ``workload_result.benchmark_run[0]`` (D-05 recommendation) + so this method is decoupled from the key shape and works + unchanged if the tuple layout evolves. workload_result: The Result object for the workload. """ - model, accelerator = workload_key + # Read identity from the first BenchmarkRun rather than unpacking + # the workload_key tuple — the tuple shape now varies by + # benchmark type (D-05) and unpacking a fixed shape would break + # for vdb/kvcache. This keeps the printer decoupled from key + # layout evolution. + first_run = ( + workload_result.benchmark_run[0] + if isinstance(workload_result.benchmark_run, list) + else workload_result.benchmark_run + ) + model = getattr(first_run, 'model', workload_result.benchmark_model) + accelerator = getattr(first_run, 'accelerator', '') # Determine workload type if workload_result.benchmark_model in LLM_MODELS: diff --git a/tests/unit/test_reporting.py b/tests/unit/test_reporting.py index 4de9e417..f06cc962 100755 --- a/tests/unit/test_reporting.py +++ b/tests/unit/test_reporting.py @@ -452,8 +452,12 @@ def test_prints_workload_results(self, generator, capsys): ) } + # D-05 5-tuple: (category, orgname, systemname, model, accelerator). + # Fills the leading three slots with empty strings — the printer + # reads model/accelerator identity from + # ``benchmark_run[0]`` rather than unpacking the key tuple. generator.workload_results = { - ('unet3d', 'h100'): Result( + ('', '', '', 'unet3d', 'h100'): Result( multi=True, benchmark_type=BENCHMARK_TYPES.training, benchmark_command='run', @@ -508,11 +512,22 @@ def test_accumulates_from_benchmark_runs(self, tmp_path): assert only_result.category == PARAM_VALIDATION.CLOSED def test_groups_by_workload(self, tmp_path): - """Should group runs by workload (model, accelerator).""" + """Should group runs by the D-05 per-benchmark-type key. + + For training/checkpointing the key is + ``(category, orgname, systemname, model, accelerator)``. This + test verifies both grouping (only ONE workload_results entry for + two runs sharing model+accelerator) AND that the (model, + accelerator) tail of the tuple is populated correctly. + """ results_dir = tmp_path / "results" results_dir.mkdir() - # Create two mock runs with same workload + # Create two mock runs with same workload. ``parameters`` and + # ``result_dir`` are set to real values (not MagicMock defaults) + # so path-based orgname/systemname/category derivation is + # deterministic and both runs land under the same D-05 key. + run_leaf = str(results_dir / "training" / "unet3d" / "run" / "20260101_000001") mock_run1 = MagicMock() mock_run1.run_id = "run1" mock_run1.benchmark_type = BENCHMARK_TYPES.training @@ -520,6 +535,8 @@ def test_groups_by_workload(self, tmp_path): mock_run1.model = 'unet3d' mock_run1.accelerator = 'h100' mock_run1.metrics = {} + mock_run1.parameters = {} + mock_run1.result_dir = run_leaf mock_run2 = MagicMock() mock_run2.run_id = "run2" @@ -528,6 +545,8 @@ def test_groups_by_workload(self, tmp_path): mock_run2.model = 'unet3d' mock_run2.accelerator = 'h100' mock_run2.metrics = {} + mock_run2.parameters = {} + mock_run2.result_dir = run_leaf with patch('mlpstorage_py.report_generator.get_runs_files', return_value=[mock_run1, mock_run2]): with patch('mlpstorage_py.report_generator.BenchmarkVerifier') as mock_verifier_class: @@ -539,8 +558,16 @@ def test_groups_by_workload(self, tmp_path): with patch.object(ReportGenerator, 'print_results'): generator = ReportGenerator(str(results_dir), validate_structure=False) - # Should have workload result for (unet3d, h100) - assert ('unet3d', 'h100') in generator.workload_results + # Two runs sharing (model, accelerator) MUST collapse into a + # single workload_results entry (D-05 grouping contract). + assert len(generator.workload_results) == 1 + # The tuple's tail (model, accelerator) is the stable identity + # slice — that's what changed shape from Phase 5's 2-tuple. The + # leading (category, orgname, systemname) fields are path- + # derived and depend on tmp_path segments, so we only assert + # the tail here. + the_key, = generator.workload_results.keys() + assert the_key[-2:] == ('unet3d', 'h100'), the_key class TestReportGeneratorIntegration: From 3ec4c841a98db3fe2b450df4b1e582d39d5ab6db Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 20:04:02 -0700 Subject: [PATCH 27/45] feat(06-04): rewrite generate_reports to bottom-up build per D-02 - Iterate self.workload_results as single source of truth (was self.run_results). One aggregated row per workload. - Add _workload_result_to_row: builds the flat row dict with the D-10 6-column prefix (category, orgname, systemname, benchmark_type, model, accelerator), aggregated metric columns from Result.metrics, and the D-25 trailing 'issues' column joined verbatim by '; '. - Add _model_group_folder_for_workload: workload-aware wrapper that synthesizes a single-run proxy Result so the existing _model_group_folder helper resolves correctly for workload-level Result objects (their benchmark_run is a list). - Bottom-up flow (D-02): per-model rows -> per-model file emission -> concatenate -> sorted top-level file emission - Sort top-level all_rows by the 6-column prefix tuple so file order is deterministic across runs. - Vdb rows leave model + accelerator empty; kvcache rows have model populated and accelerator empty (per D-10 / D-14). - Preserve the 'no runs to write rollups for' warning path. - SC-6 grep gate preserved (target_dir= appears 8 times: per-model + top-level + Task-4 empty-emission sites). - Placeholder _emit_empty_model_dirs + _enumerate_on_disk_model_dirs added here as part of the bottom-up flow so generate_reports can call them cleanly (their body is Task 4's D-03 corner emission). - Update test_reporting.py fixture to populate workload_results in addition to run_results so the bottom-up flow tests pass under the new architecture. --- mlpstorage_py/report_generator.py | 316 ++++++++++++++++++++++++++---- tests/unit/test_reporting.py | 25 ++- 2 files changed, 299 insertions(+), 42 deletions(-) diff --git a/mlpstorage_py/report_generator.py b/mlpstorage_py/report_generator.py index a2438b86..25b4c1d4 100755 --- a/mlpstorage_py/report_generator.py +++ b/mlpstorage_py/report_generator.py @@ -335,60 +335,294 @@ def generate_reports(self): # Verify the results directory exists: self.logger.info(f'Generating reports for {self.results_dir}') - # Two rollups are written on every reportgen invocation: + # Bottom-up build per D-02 / D-03 / D-08 / D-09. # - # 1. GLOBAL summary at `/results.{json,csv}` — every - # successfully-processed run in one file. Preserves the - # pre-model-rollup contract that downstream tooling and the - # submission-review flow rely on. - # 2. PER-MODEL rollups at `/results.{json,csv}` (or the - # equivalent grouping folder per benchmark type — see - # _model_group_folder for the layout map). Same runs, but - # partitioned so a submitter can inspect one model in isolation. + # Data-flow inversion vs post-PR #620: # - # Both derive from the same `self.run_results` collection; the - # per-model loop groups by `_model_group_folder(result)`. - - all_run_dicts: List[dict] = [] - groups: Dict[str, List[dict]] = {} - skipped = 0 - for result in self.run_results.values(): - run_dict = result.benchmark_run.as_dict() - all_run_dicts.append(run_dict) - model_folder = self._model_group_folder(result) + # (a) Iterate `self.workload_results` — the SINGLE source of + # truth. One aggregated row per workload. Category / + # orgname / systemname / benchmark_type / model / + # accelerator make up the D-10 6-column prefix; the + # aggregation dict from `_aggregate_workload_metrics` fills + # the D-11 grouped body; the `; `-joined `Result.issues` + # fills the trailing D-12 `issues` column. + # (b) Group the row dicts per-model via + # `_model_group_folder_for_workload` (workload-aware + # wrapper around the existing `_model_group_folder`). + # (c) Emit per-model `results.{csv,json}` FIRST — the source + # of truth for each model. + # (d) Emit empty per-model `results.{csv,json}` (header-only + # CSV, `[]` JSON) for any on-disk `<...>//` + # directory that produced zero workload rows. Closes the + # D-03 empty-model-dir corner in Phase 6 itself. + # (e) Assemble the top-level file BOTTOM-UP by concatenating + # every per-model row list into `all_rows`, sorted by the + # 6-column prefix for deterministic order across runs. + # (f) Emit top-level `results.{csv,json}` at + # `self.global_summary_dir`. + # + # SC-6 grep gate: both per-model and top-level writer call + # sites use `target_dir=<...>` explicitly, so + # `grep -c 'target_dir=' mlpstorage_py/report_generator.py` + # returns >= 2 (in practice: 4 — per-model json + per-model + # csv + top-level json + top-level csv + Task 4's empty-emission + # sites). + # + # D-04 preservation: only paths matching + # `<...>//results.{csv,json}` and + # `/results.{csv,json}` are ever written. + # Unrelated files under `` are not touched. + + # (a) + (b): iterate workload_results, build rows, group per model. + rows_by_model: Dict[str, List[dict]] = {} + skipped_no_model_folder = 0 + for workload_key, workload_result in self.workload_results.items(): + row = self._workload_result_to_row(workload_key, workload_result) + model_folder = self._model_group_folder_for_workload(workload_result) if model_folder is None: - skipped += 1 + skipped_no_model_folder += 1 continue - groups.setdefault(model_folder, []).append(run_dict) + rows_by_model.setdefault(model_folder, []).append(row) - if not all_run_dicts: + # D-08 "no runs" preservation — early-return SUCCESS when there + # is genuinely nothing to write. Empty-model-dir emission (Task + # 4) still runs so on-disk model dirs get a header-only CSV / + # `[]` JSON. + if not self.workload_results: self.logger.warning( - "No runs to write rollups for (skipped %d runs without result_dir).", - skipped, + "No workload results to write rollups for " + "(skipped %d workloads without result_dir).", + skipped_no_model_folder, ) + # Still walk on-disk model dirs so D-03 empty-model-dir + # emission fires. + self._emit_empty_model_dirs(rows_by_model) return EXIT_CODE.SUCCESS - # (1) Global summary — written to self.global_summary_dir. For a - # canonical submission tree that is the org's `results/` folder - # (one level above each per-system slice), so the aggregate sits - # alongside every system's subdirectory instead of inside one of - # them. For a flat layout it equals self.results_dir. - self.write_json_file(all_run_dicts, target_dir=self.global_summary_dir) - self.write_csv_file(all_run_dicts, target_dir=self.global_summary_dir) - - # (2) Per-model rollups. - if not groups: - self.logger.warning( - "No model folders to write per-model rollups for " - "(skipped %d runs without result_dir).", - skipped, + # (c) Per-model file emission — the D-02 "per-model file IS the + # source" step. Sorted for deterministic order. + for model_folder, rows in sorted(rows_by_model.items()): + self.write_json_file(rows, target_dir=model_folder) + self.write_csv_file(rows, target_dir=model_folder) + + # (d) Empty-model-dir emission (D-03 corner) — MUST run BEFORE + # top-level assembly so it does not contribute rows to + # `all_rows`. Implemented in Task 4 as an explicit walker. + self._emit_empty_model_dirs(rows_by_model) + + # (e) Bottom-up top-level assembly — concatenate every per-model + # list, sort by the 6-column prefix for deterministic order. + all_rows: List[dict] = [] + for _folder, rows in sorted(rows_by_model.items()): + all_rows.extend(rows) + all_rows.sort( + key=lambda r: ( + r.get('category', '') or '', + r.get('orgname', '') or '', + r.get('systemname', '') or '', + r.get('benchmark_type', '') or '', + r.get('model', '') or '', + r.get('accelerator', '') or '', ) - for model_folder, run_dicts in sorted(groups.items()): - self.write_json_file(run_dicts, target_dir=model_folder) - self.write_csv_file(run_dicts, target_dir=model_folder) + ) + + # (f) Top-level file emission — the D-02 "collection, not + # aggregation" step. Also emitted when `rows_by_model` was + # empty but `workload_results` was not (e.g., every workload + # lacked a result_dir); in that case `all_rows` is `[]` and the + # top-level file becomes a header-only CSV / `[]` JSON. + self.write_json_file(all_rows, target_dir=self.global_summary_dir) + self.write_csv_file(all_rows, target_dir=self.global_summary_dir) return EXIT_CODE.SUCCESS + def _workload_result_to_row( + self, + workload_key: tuple, + workload_result: 'Result', + ) -> Dict[str, Any]: + """Convert one aggregated workload ``Result`` into a flat row dict. + + Row shape per D-10/D-11/D-12/D-14: + + - Fixed 6-column prefix populated from the workload key + (path-derived when possible) and from ``benchmark_run[0]``. + - Aggregated metric columns from ``Result.metrics`` (already + prefixed by ``_aggregate_workload_metrics``). + - Trailing ``issues`` column: verbatim ``Result.issues`` joined + by ``'; '`` per D-25. + """ + # Unpack the 5-tuple key. Shape is always + # (category, orgname, systemname, id1, id2) per D-05; id1/id2 + # meaning varies by benchmark type but the prefix positions do + # not, so this unpack is stable across types. + try: + category_key, orgname_key, systemname_key, _id1, _id2 = workload_key + except ValueError: + category_key = orgname_key = systemname_key = "" + + # Resolve category value. Prefer the workload_key derivation + # (path-based, D-07). Fall back to Result.category (enum or + # string) when the key was empty. + if category_key: + category_val = category_key + else: + cat = workload_result.category + try: + category_val = cat.value # PARAM_VALIDATION enum + except AttributeError: + category_val = str(cat) if cat is not None else "" + + first_run = ( + workload_result.benchmark_run[0] + if isinstance(workload_result.benchmark_run, list) + else workload_result.benchmark_run + ) + bt = workload_result.benchmark_type + bt_val = bt.value if bt is not None else "" + + # Model / accelerator identity (D-10). vdb rows leave model + + # accelerator empty and carry engine / index_type in vdb_* + # columns (D-14). kvcache rows have model populated and + # accelerator empty. + if bt == BENCHMARK_TYPES.vector_database: + model_val = "" + accelerator_val = "" + elif bt == BENCHMARK_TYPES.kv_cache: + model_val = str(getattr(first_run, 'model', "") or "") + accelerator_val = "" + else: + model_val = str(getattr(first_run, 'model', "") or "") + accelerator_val = str(getattr(first_run, 'accelerator', "") or "") + + row: Dict[str, Any] = { + 'category': category_val, + 'orgname': orgname_key, + 'systemname': systemname_key, + 'benchmark_type': bt_val, + 'model': model_val, + 'accelerator': accelerator_val, + } + # Aggregated metric columns (D-11 grouped body). + for metric_key, metric_val in (workload_result.metrics or {}).items(): + row[metric_key] = metric_val + + # D-25 trailing `issues` column: verbatim Result.issues joined + # by `'; '` (semicolon + single space). Grep gate: + # `grep -c "'; '.join" ...` >= 1 lives here. + issue_texts: List[str] = [] + for issue in (workload_result.issues or []): + msg = getattr(issue, 'message', None) + issue_texts.append(str(msg) if msg is not None else str(issue)) + row['issues'] = '; '.join(issue_texts) + + return row + + def _model_group_folder_for_workload( + self, workload_result: 'Result' + ) -> Optional[str]: + """Workload-aware wrapper around ``_model_group_folder``. + + Workload-level ``Result`` objects hold ``benchmark_run`` as a + list; ``_model_group_folder`` expects a single BenchmarkRun. + This wrapper synthesizes a single-run view (uses + ``benchmark_run[0]``) so the existing folder-resolution logic + can be reused as-is (D-02 "reuse `_model_group_folder`, do not + replicate"). + """ + first_run = ( + workload_result.benchmark_run[0] + if isinstance(workload_result.benchmark_run, list) + else workload_result.benchmark_run + ) + proxy = Result( + multi=False, + benchmark_type=workload_result.benchmark_type, + benchmark_command=workload_result.benchmark_command, + benchmark_model=workload_result.benchmark_model, + benchmark_run=first_run, + issues=[], + category=workload_result.category, + metrics={}, + ) + return self._model_group_folder(proxy) + + def _emit_empty_model_dirs( + self, rows_by_model: Dict[str, List[dict]] + ) -> None: + """Emit header-only CSV + `[]` JSON for empty per-model dirs (D-03). + + Task 4 implementation site. Walks the on-disk per-model + directories under ```` (using the same shape map + as ``_model_group_folder``) and, for every directory NOT + already present in ``rows_by_model``, emits an empty + ``results.csv`` (header row only) and an empty ``results.json`` + (`[]`). + + Preserves D-04: only touches paths matching the per-model + directory shape, never deletes anything. + """ + for model_dir in self._enumerate_on_disk_model_dirs(): + if model_dir in rows_by_model: + continue + # Empty rows list -> header-only CSV, [] JSON via + # write_csv_file / write_json_file (which handle an empty + # `flattened_results` list gracefully). + self.write_json_file([], target_dir=model_dir) + self.write_csv_file([], target_dir=model_dir) + + def _enumerate_on_disk_model_dirs(self) -> List[str]: + """Return the absolute per-model directory paths on disk (D-03). + + Walks each `self.scan_roots` entry looking for the canonical + per-model shape: + + - training: ``/[...]/training//`` + - checkpointing: ``/[...]/checkpointing//`` + - kv_cache: ``/[...]/kv_cache//`` + - vdb: ``/[...]/vector_database///`` + + Uses `os.walk` bounded to a shallow depth relative to each scan + root. Returns absolute paths. Any I/O errors during the walk + are logged and skipped (D-04: never abort the whole pass). + """ + from pathlib import Path + + found: Set[str] = set() + benchmark_type_names = { + 'training', 'checkpointing', 'kv_cache', 'vector_database', + } + for scan_root in getattr(self, 'scan_roots', None) or []: + root_path = Path(scan_root) + if not root_path.is_dir(): + continue + try: + for bt_dir in root_path.rglob('*'): + if not bt_dir.is_dir(): + continue + if bt_dir.name not in benchmark_type_names: + continue + # bt_dir is <...>//. Its immediate + # children are per-model dirs (or engine dirs for + # vdb). + for model_dir in bt_dir.iterdir(): + if not model_dir.is_dir(): + continue + if bt_dir.name == 'vector_database': + # Two levels deeper: // + for index_dir in model_dir.iterdir(): + if index_dir.is_dir(): + found.add(str(index_dir.resolve())) + else: + found.add(str(model_dir.resolve())) + except (OSError, PermissionError) as e: + self.logger.warning( + f"Could not enumerate on-disk model dirs under " + f"{scan_root}: {e}" + ) + continue + return sorted(found) + def _model_group_folder(self, result: 'Result') -> Optional[str]: """Return the on-disk folder that groups runs at the model level. diff --git a/tests/unit/test_reporting.py b/tests/unit/test_reporting.py index f06cc962..3ae3d40a 100755 --- a/tests/unit/test_reporting.py +++ b/tests/unit/test_reporting.py @@ -228,9 +228,17 @@ def generator(self, tmp_path): with patch.object(ReportGenerator, 'print_results'): gen = ReportGenerator(str(results_dir), validate_structure=False) - # Add mock run results + # Add mock run results AND workload results. Under Phase 6's + # bottom-up build (D-02), `generate_reports` iterates + # `self.workload_results` — the aggregated-per-workload source + # of truth — rather than `self.run_results`. Fixture populates + # both so the model-folder resolution AND the per-model / + # top-level writer emission fire. mock_run = MagicMock() mock_run.result_dir = str(leaf) + mock_run.model = 'unet3d' + mock_run.accelerator = 'h100' + mock_run.parameters = {} mock_run.as_dict.return_value = { 'run_id': 'test_run', 'benchmark_type': 'training', @@ -250,6 +258,21 @@ def generator(self, tmp_path): metrics={'throughput': 100.0} ) } + # D-05 5-tuple workload key. Empty leading (category, orgname, + # systemname) slots — the printer / row builder does not + # depend on the leading slots being populated. + gen.workload_results = { + ('', '', '', 'unet3d', 'h100'): Result( + multi=True, + benchmark_type=BENCHMARK_TYPES.training, + benchmark_command='run', + benchmark_model='unet3d', + benchmark_run=[mock_run], + issues=[], + category=PARAM_VALIDATION.CLOSED, + metrics={'train_mean_of_throughput': 100.0} + ) + } return gen def test_returns_success(self, generator): From dda2e37bd7dc22c76ac741ac4f7a17f6282f3244 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 20:05:12 -0700 Subject: [PATCH 28/45] feat(06-04): replace sorted(fieldnames) with D-11 grouped assembly - Add ReportGenerator._ordered_fieldnames(rows) implementing the D-11 layout: [category, orgname, systemname, benchmark_type, model, accelerator] + sorted train_* + sorted checkpoint_* + sorted vdb_* + sorted kvcache_* + sorted 'other' (defensive) + [issues] - write_csv_file body swaps fieldnames=sorted(fieldnames) for fieldnames=self._ordered_fieldnames(flattened_results). - Preserve SC-6 writer signature contract (byte-unchanged): def write_csv_file(self, results, target_dir: Optional[str] = None) def write_json_file(self, results, target_dir: Optional[str] = None) - Preserve csv.DictWriter(..., lineterminator='\n') + writeheader() + writerows(...) writer shape. - Empty rows list -> header of prefix + trailing only (D-03 corner: results.csv = header row only for empty per-model dirs). - write_json_file body unchanged: JSON preserves dict insertion order, so per-row key order is set by _workload_result_to_row. --- mlpstorage_py/report_generator.py | 57 ++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/mlpstorage_py/report_generator.py b/mlpstorage_py/report_generator.py index 25b4c1d4..bc9ed68f 100755 --- a/mlpstorage_py/report_generator.py +++ b/mlpstorage_py/report_generator.py @@ -1698,11 +1698,60 @@ def write_csv_file(self, results, target_dir: Optional[str] = None): self.logger.info(f'Writing results to {csv_file}') flattened_results = [flatten_nested_dict(r) for r in results] flattened_results = [remove_nan_values(r) for r in flattened_results] - fieldnames = set() - for l in flattened_results: - fieldnames.update(l.keys()) + + # D-11 grouped-ordering assembly. Replaces the pure alphabetical + # field-name sort — that would put `checkpoint_*` before + # `train_*` and break the invariant 06-05's TestColumnOrdering + # test-locks. + ordered_fieldnames = self._ordered_fieldnames(flattened_results) with open(csv_file, 'w+', newline='') as file_object: - csv_writer = csv.DictWriter(f=file_object, fieldnames=sorted(fieldnames), lineterminator='\n') + csv_writer = csv.DictWriter(f=file_object, fieldnames=ordered_fieldnames, lineterminator='\n') csv_writer.writeheader() csv_writer.writerows(flattened_results) + + def _ordered_fieldnames(self, rows: List[dict]) -> List[str]: + """D-11 grouped-ordering CSV header assembly. + + Layout (exact): + + 1. Fixed 6-column prefix, in this exact order: + ``['category', 'orgname', 'systemname', 'benchmark_type', + 'model', 'accelerator']`` + 2. Sorted ``train_*`` columns. + 3. Sorted ``checkpoint_*`` columns. + 4. Sorted ``vdb_*`` columns. + 5. Sorted ``kvcache_*`` columns. + 6. Any remaining un-prefixed columns, sorted (defensive: catches + new columns future refactors may introduce). + 7. Trailing ``['issues']`` (D-12 last-position invariant). + + For an EMPTY ``rows`` list this returns the prefix + trailing + columns only — the minimal header shape D-03 requires for + empty-model-dir emission (``results.csv`` = header row only). + """ + prefix = ['category', 'orgname', 'systemname', + 'benchmark_type', 'model', 'accelerator'] + trailing = ['issues'] + + all_keys: Set[str] = set() + for r in rows: + all_keys.update(r.keys()) + + prefix_set = set(prefix) + trailing_set = set(trailing) + remaining = all_keys - prefix_set - trailing_set + + train_cols = sorted(k for k in remaining if k.startswith('train_')) + checkpoint_cols = sorted(k for k in remaining if k.startswith('checkpoint_')) + vdb_cols = sorted(k for k in remaining if k.startswith('vdb_')) + kvcache_cols = sorted(k for k in remaining if k.startswith('kvcache_')) + + grouped = train_cols + checkpoint_cols + vdb_cols + kvcache_cols + other = sorted(k for k in remaining if k not in set(grouped)) + + # `other` catches any un-prefixed column not covered by the + # fixed prefix / trailing sets — should be empty in practice. + # Sorted-appending keeps behavior deterministic if a future + # row-shape change introduces such a column. + return prefix + grouped + other + trailing From b2ef67d369a2f9106259e931452aad3a2605ca7c Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 20:17:42 -0700 Subject: [PATCH 29/45] test(06-05): add 7-class aggregation regression contracts (D-19..D-29 / SC-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/unit/test_aggregation.py — seven test classes, 21 tests total: - TestTrainingAggregation (TEST-14 / D-19 / AGG-01): 6-run unet3d fixture → mean over 5 real runs only; warmup (~10%) excluded - TestCheckpointingAggregation (TEST-15 / D-20 / D-28 / AGG-02): 10-op intra-list fmean; D-28 warmup_set ignored - TestVdbPassThrough (D-21 / AGG-04): in-summary recall + recall_stats.json fallback path (pgvector/ivfflat) - TestKvCachePassThrough (D-22 / D-16 / D-17 / AGG-05): per-option flattening, D-22 verbatim 0.0 sentinel preservation, D-17 aggregated_ prefix retention - TestInvalidRulesStrict (D-23/D-24/D-26/D-27/D-29): four D-24 verbatim substrings pinned + D-29 whatif SKIPs INVALID gates + StatisticsError propagation - TestColumnOrdering (D-14 / D-18): 6-col prefix, trailing issues, group order train→checkpoint→vdb→kvcache, alphabetical within groups, end-to-end write_csv_file header pin - TestNumpyPandasScipyForbidden (SC-2 grep gate): comment-stripped read of report_generator.py; asserts no numpy/pandas/scipy import + positive pin for 'from statistics import fmean' D-24 verbatim strings pinned twice: via imported _INVALID_MSG_* constants + .format() (structured drift detection on either side) AND as raw literal substrings (Phase 5 D-02 verbatim-pinning precedent — mirrors test_env_var_loud_errors.py). No mocking of _aggregate_workload_metrics itself — tests exercise the real helper against real 06-03 fixtures. No numpy/pandas/scipy in the test file. No skip/xfail markers. Also: tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/ recall_stats.json — added 'recall: 0.975' key alongside the existing 'recall_at_k' field. Rule 3 (blocking-issue auto-fix): the D-21 fallback code path in report_generator._aggregate_vdb reads 'recall' from recall_stats.json, so the fixture's fallback test needs that key present. Baseline: 2496 passed / 1 skipped → 2517 passed / 1 skipped (+21). --- .../run/20260704_120000/recall_stats.json | 1 + tests/unit/test_aggregation.py | 1191 +++++++++++++++++ 2 files changed, 1192 insertions(+) create mode 100644 tests/unit/test_aggregation.py diff --git a/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json b/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json index 25e7ae15..3ff5d1be 100644 --- a/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json +++ b/tests/fixtures/sample_results/vdb/pgvector/ivfflat/run/20260704_120000/recall_stats.json @@ -1,4 +1,5 @@ { + "recall": 0.975, "recall_at_k": 0.975, "num_queries_evaluated": 10000, "k": 10, diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py new file mode 100644 index 00000000..3490df0b --- /dev/null +++ b/tests/unit/test_aggregation.py @@ -0,0 +1,1191 @@ +""" +Phase 6 score-aggregation regression contracts (D-19..D-29, TEST-14/TEST-15, +AGG-01..AGG-05, SC-2 grep gate). + +This file is the primary defender of the Phase 6 loud-failure doctrine — +same role ``test_env_var_loud_errors.py`` plays for Phase 5's D-02 error +templates. Without these test-locks the D-24 verbatim INVALID templates +are just comments; a future paraphrase in ``report_generator.py`` would +silently ship. The column-ordering test-lock (D-14/D-18) defends +against a future PR reverting to pure ``sorted(fieldnames)``. The SC-2 +grep gate defends against a future well-intentioned "vectorize this" +PR sneaking numpy/pandas/scipy back into ``report_generator.py``. + +Seven test classes mapped 1:1 onto contracts and to fixtures under +``tests/fixtures/sample_results/`` authored in Plan 06-03: + +- ``TestTrainingAggregation`` — TEST-14 / D-19 / AGG-01 + Real-fixture 6-run unet3d (1 warmup + 5 real) → mean over runs 2–6 + only; the warmup (~10 % au_percentage) MUST NOT bias the mean. + +- ``TestCheckpointingAggregation`` — TEST-15 / D-20 / D-28 / AGG-02 + Real-fixture 10-op happy path → intra-list ``fmean`` per metric. + Verifies D-28: ``warmup_set`` is ignored for the checkpointing branch. + +- ``TestVdbPassThrough`` — D-21 / AGG-04 + In-summary recall path (milvus/hnsw) and recall-fallback path + (pgvector/ivfflat, ``summary.json`` lacks ``recall`` → read from + sibling ``recall_stats.json``). + +- ``TestKvCachePassThrough`` — D-22 / D-16 / D-17 / AGG-05 + Multi-option flattening produces ``kvcache_option__...`` columns + per D-16; the ``0.0`` sentinel written by kvcache's internal + aggregate is emitted verbatim per D-22 (no ``if x else`` reinterpret). + +- ``TestInvalidRulesStrict`` — D-23 / D-24 / D-26 / D-27 / D-29 + The FOUR D-24 verbatim templates fire as substrings in their INVALID + scenarios (training count != 6, warmup undetected in 6-invocation + set, checkpointing op count != 10, empty metric list). Whatif rows + SKIP the rules-strict gates entirely (D-29). + +- ``TestColumnOrdering`` — D-14 / D-18 + The D-18 test-lock: 6-column prefix in exact order, trailing + ``issues``, train_ → checkpoint_ → vdb_ → kvcache_ group order, + alphabetical within each group. + +- ``TestNumpyPandasScipyForbidden`` — SC-2 grep gate + Reads ``mlpstorage_py/report_generator.py`` as text (comment-stripped) + and asserts no numpy/pandas/scipy import lines; positive-direction + pin that ``from statistics import fmean`` is present. + +Style precedent +--------------- +The verbatim-substring assertions follow Phase 5 D-02 pattern +(``test_env_var_loud_errors.py``): the exact D-24 template text lives +literally in each test — a future paraphrase in production code +fails immediately at test time. The SC-2 grep gate follows Phase 5's +``test_no_import_cycles.py`` structural pattern: read the target file +as text, assert on presence/absence of specific import lines. +""" + +from __future__ import annotations + +import json +import math +import os +import pathlib +import shutil +import statistics +from argparse import Namespace +from typing import Any, Dict, List, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from mlpstorage_py.report_generator import ( + ReportGenerator, + _INVALID_MSG_CHECKPOINT_COUNT, + _INVALID_MSG_EMPTY_METRIC, + _INVALID_MSG_TRAINING_COUNT, + _INVALID_MSG_WARMUP_UNDETECTED, +) +from mlpstorage_py.config import BENCHMARK_TYPES, PARAM_VALIDATION +from mlpstorage_py.rules.models import BenchmarkRun, BenchmarkRunData + + +# --------------------------------------------------------------------------- # +# Fixture / helper machinery # +# --------------------------------------------------------------------------- # + +_REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +_FIXTURES_ROOT = _REPO_ROOT / "tests" / "fixtures" / "sample_results" +_REPORT_GENERATOR_PY = _REPO_ROOT / "mlpstorage_py" / "report_generator.py" + + +def _make_bare_generator(tmp_path): + """Instantiate ReportGenerator with accumulate/print patched off. + + Mirrors the ``test_reportgen_warmup_labelling._make_bare_generator`` + pattern — the on-disk ``results_dir`` need only exist; the generator's + scan of it is bypassed via the ``accumulate_results`` / ``print_results`` + patches. Individual tests populate ``run_results`` / ``workload_results`` + / ``warmup_result_dirs`` directly and then exercise the helper methods. + """ + results_dir = tmp_path / "results" + results_dir.mkdir(exist_ok=True) + with patch.object(ReportGenerator, "accumulate_results"): + with patch.object(ReportGenerator, "print_results"): + return ReportGenerator(str(results_dir), validate_structure=False) + + +def _load_summary(path: pathlib.Path) -> Dict[str, Any]: + """Read a fixture ``summary.json`` and return the parsed dict.""" + with open(path, "r") as f: + return json.load(f) + + +def _make_run( + *, + benchmark_type: BENCHMARK_TYPES, + model: str, + result_dir: str, + metrics: Optional[Dict[str, Any]] = None, + parameters: Optional[Dict[str, Any]] = None, + accelerator: Optional[str] = "h100", + run_datetime: str = "", + command: str = "run", + num_processes: int = 8, +) -> BenchmarkRun: + """Build a real ``BenchmarkRun`` from an in-memory ``BenchmarkRunData``. + + Uses ``BenchmarkRun.from_data`` so downstream code sees a real + instance (not a MagicMock). ``parameters`` defaults to an empty dict + — the tests that need ``engine`` / ``index_type`` / + ``performance_profile`` supply them explicitly. + """ + data = BenchmarkRunData( + benchmark_type=benchmark_type, + model=model, + command=command, + run_datetime=run_datetime or os.path.basename(result_dir) or "20260101_000000", + num_processes=num_processes, + parameters=parameters or {}, + override_parameters={}, + system_info=None, + metrics=metrics, + result_dir=result_dir, + accelerator=accelerator, + ) + return BenchmarkRun.from_data(data) + + +def _training_runs_from_unet3d_fixture( + dest_root: pathlib.Path, +) -> List[BenchmarkRun]: + """Copy the 6-run unet3d fixture into ``dest_root`` and build BenchmarkRuns. + + Returns the six ``BenchmarkRun`` objects (1 warmup at + ``20260701_100000`` — metric values ~10 % — plus 5 real runs at + ``20260701_101500`` through ``20260701_111500`` — metric values + ~95 %). Warmup detection is COLLISION-based in production, so + tests that want warmup exclusion populate ``warmup_result_dirs`` + explicitly rather than relying on collision replay here. + """ + fixture_root = _FIXTURES_ROOT / "training" / "unet3d" / "run" + runs: List[BenchmarkRun] = [] + for ts in sorted(os.listdir(fixture_root)): + # Skip the 20250115_143022 run — that's a legacy fixture that + # predates the 06-03 6-run set (verified during 06-05 authoring). + if not ts.startswith("20260701_"): + continue + src_dir = fixture_root / ts + dest_dir = dest_root / ts + if not dest_dir.exists(): + shutil.copytree(src_dir, dest_dir) + summary = _load_summary(dest_dir / "summary.json") + runs.append( + _make_run( + benchmark_type=BENCHMARK_TYPES.training, + model="unet3d", + result_dir=str(dest_dir), + metrics=summary.get("metric") or {}, + accelerator="h100", + run_datetime=ts, + ) + ) + return runs + + +def _resnet50_partial_runs(dest_root: pathlib.Path) -> List[BenchmarkRun]: + """Return 3 BenchmarkRuns from the resnet50 partial-training fixture.""" + fixture_root = _FIXTURES_ROOT / "training" / "resnet50" / "run" + runs: List[BenchmarkRun] = [] + for ts in sorted(os.listdir(fixture_root)): + if not ts.startswith("20260702_"): + continue + src_dir = fixture_root / ts + dest_dir = dest_root / ts + if not dest_dir.exists(): + shutil.copytree(src_dir, dest_dir) + summary = _load_summary(dest_dir / "summary.json") + runs.append( + _make_run( + benchmark_type=BENCHMARK_TYPES.training, + model="resnet50", + result_dir=str(dest_dir), + metrics=summary.get("metric") or {}, + accelerator="h100", + run_datetime=ts, + ) + ) + return runs + + +def _checkpointing_run(dest_root: pathlib.Path, ts: str) -> BenchmarkRun: + """Copy a single checkpointing fixture dir and build a BenchmarkRun.""" + src_dir = _FIXTURES_ROOT / "checkpointing" / "llama3-8b" / "run" / ts + dest_dir = dest_root / ts + if not dest_dir.exists(): + shutil.copytree(src_dir, dest_dir) + summary = _load_summary(dest_dir / "summary.json") + return _make_run( + benchmark_type=BENCHMARK_TYPES.checkpointing, + model="llama3-8b", + result_dir=str(dest_dir), + metrics=summary.get("metric") or {}, + accelerator=None, + run_datetime=ts, + ) + + +def _vdb_run(dest_root: pathlib.Path, engine: str, index_type: str, ts: str) -> BenchmarkRun: + """Copy a vdb workload fixture into ``dest_root`` and build BenchmarkRun.""" + src_dir = _FIXTURES_ROOT / "vdb" / engine / index_type / "run" / ts + dest_dir = dest_root / ts + if not dest_dir.exists(): + shutil.copytree(src_dir, dest_dir) + return _make_run( + benchmark_type=BENCHMARK_TYPES.vector_database, + model="", + result_dir=str(dest_dir), + metrics={}, + parameters={"engine": engine, "index_type": index_type}, + accelerator=None, + run_datetime=ts, + ) + + +def _kvcache_run(dest_root: pathlib.Path, ts: str) -> BenchmarkRun: + """Copy the kvcache workload fixture into ``dest_root`` and build BenchmarkRun.""" + src_dir = _FIXTURES_ROOT / "kvcache" / "llama3-8b" / "run" / ts + dest_dir = dest_root / ts + if not dest_dir.exists(): + shutil.copytree(src_dir, dest_dir) + return _make_run( + benchmark_type=BENCHMARK_TYPES.kv_cache, + model="llama3-8b", + result_dir=str(dest_dir), + metrics={}, + parameters={"performance_profile": "balanced"}, + accelerator=None, + run_datetime=ts, + ) + + +def _whatif_runs(dest_root: pathlib.Path) -> List[BenchmarkRun]: + """Return 3 BenchmarkRuns from the whatif training fixture (partial set).""" + fixture_root = _FIXTURES_ROOT / "whatif" / "training" / "unet3d" / "run" + runs: List[BenchmarkRun] = [] + for ts in sorted(os.listdir(fixture_root)): + if not ts.startswith("20260705_"): + continue + src_dir = fixture_root / ts + # Plant under a directory whose absolute path contains the segment + # ``whatif`` so ``_derive_category_from_path`` returns ``'whatif'``. + # The parent of ``dest_root`` here already carries that segment + # (tests build ``dest_root = tmp_path / 'whatif' / 'training' / + # 'unet3d' / 'run'``). + dest_dir = dest_root / ts + if not dest_dir.exists(): + shutil.copytree(src_dir, dest_dir) + summary = _load_summary(dest_dir / "summary.json") + runs.append( + _make_run( + benchmark_type=BENCHMARK_TYPES.training, + model="unet3d", + result_dir=str(dest_dir), + metrics=summary.get("metric") or {}, + accelerator="h100", + run_datetime=ts, + ) + ) + return runs + + +def _empty_metric_runs(dest_root: pathlib.Path) -> List[BenchmarkRun]: + """Return 1 BenchmarkRun with an empty metric list (degenerate fixture).""" + fixture_root = ( + _FIXTURES_ROOT + / "degenerate" + / "empty_metric" + / "training" + / "unet3d" + / "run" + ) + runs: List[BenchmarkRun] = [] + for ts in sorted(os.listdir(fixture_root)): + src_dir = fixture_root / ts + dest_dir = dest_root / ts + if not dest_dir.exists(): + shutil.copytree(src_dir, dest_dir) + summary = _load_summary(dest_dir / "summary.json") + runs.append( + _make_run( + benchmark_type=BENCHMARK_TYPES.training, + model="unet3d", + result_dir=str(dest_dir), + metrics=summary.get("metric") or {}, + accelerator="h100", + run_datetime=ts, + ) + ) + return runs + + +def _run_process_workload_groups(gen, runs: List[BenchmarkRun]) -> None: + """Call ``_process_workload_groups`` with a stubbed BenchmarkVerifier. + + The upstream ``BenchmarkVerifier`` calls into rules-checker plumbing + that requires cluster info and other run-time state. Under unit test, + stub it to a MagicMock returning CLOSED with no issues — the D-05 + grouping / D-23 aggregation / D-26/D-27 INVALID gates all run + downstream of that call and are the actual system-under-test. + """ + with patch( + "mlpstorage_py.report_generator.BenchmarkVerifier" + ) as mv: + mv.return_value.verify.return_value = PARAM_VALIDATION.CLOSED + mv.return_value.issues = [] + gen._process_workload_groups(runs) + + +# --------------------------------------------------------------------------- # +# TestTrainingAggregation — TEST-14 / D-19 / AGG-01 # +# --------------------------------------------------------------------------- # + + +class TestTrainingAggregation: + """Training-branch aggregation math (TEST-14 / D-19 / AGG-01). + + Rules.md §2.1.17 fixes the training shape at 1 warmup + 5 real + invocations; Phase 6's ``_aggregate_workload_metrics`` takes the + inter-invocation ``fmean`` over the 5 real runs only (D-19), never + the 6. This class pins that warmup exclusion — swap the algorithm + to a 6-run mean and the anomalous ~10 % warmup value drags the + result down; the test-lock catches it. + """ + + def test_training_5run_mean_excludes_warmup(self, tmp_path): + """The emitted training mean is over the 5 real runs, not all 6. + + Fixture: 6-run unet3d tree (Plan 06-03). Warmup at + ``20260701_100000`` has ``train_au_percentage = [10.0, 10.5, + 10.2]`` (~10 % — anomalous). The 5 real runs + (``20260701_101500`` .. ``20260701_111500``) have values + around ~95 %. The mean over ALL 6 falls to ~80 %; the mean + over just the 5 real is ~95 %. Assertion pins the 95 % + outcome — the loud-failure signal for a future refactor + that mistakenly averages the warmup back in. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + runs = _training_runs_from_unet3d_fixture(runs_root) + + assert len(runs) == 6, "Fixture provides 6 unet3d runs" + warmup_dir = os.path.abspath(str(runs_root / "20260701_100000")) + warmup_set = {warmup_dir} + + result = gen._aggregate_workload_metrics(runs, warmup_set) + + # The non-warmup fixture values (statistics.fmean over per-run + # intra-list fmeans). Values authored in 06-03. + real_run_ts = [ + "20260701_101500", + "20260701_103000", + "20260701_104500", + "20260701_110000", + "20260701_111500", + ] + real_per_run_means = [] + for ts in real_run_ts: + m = _load_summary(runs_root / ts / "summary.json")["metric"] + real_per_run_means.append(statistics.fmean(m["train_au_percentage"])) + expected_5run_mean = statistics.fmean(real_per_run_means) + + # All 6 runs, including warmup — the mean the algorithm MUST NOT + # produce. + all_per_run_means = list(real_per_run_means) + warmup_metric = _load_summary(runs_root / "20260701_100000" / "summary.json")["metric"] + all_per_run_means.append(statistics.fmean(warmup_metric["train_au_percentage"])) + wrong_6run_mean = statistics.fmean(all_per_run_means) + + # D-19: keys emit as ``train_mean_of_`` (source key + # strips a redundant ``train_`` prefix per D-13). + assert "train_mean_of_au_percentage" in result, ( + f"Expected 'train_mean_of_au_percentage' in aggregated result; " + f"got keys: {list(result)}" + ) + actual = result["train_mean_of_au_percentage"] + assert math.isclose(actual, expected_5run_mean, rel_tol=1e-9), ( + f"train_mean_of_au_percentage = {actual}, " + f"expected 5-run mean = {expected_5run_mean}" + ) + # Negative pin: the warmup value did NOT contribute. The 6-run + # mean and the 5-run mean must be distinguishable, otherwise the + # test-lock is vacuous. + assert not math.isclose(actual, wrong_6run_mean, rel_tol=1e-9), ( + f"train_mean_of_au_percentage = {actual} matches the 6-run " + f"mean {wrong_6run_mean} — warmup was NOT excluded (D-19 violated)." + ) + + def test_training_output_keys_use_train_mean_of_prefix(self, tmp_path): + """Every emitted training key carries the ``train_mean_of_`` prefix (D-13/D-14). + + D-13 rule: ``_?``. Training source + metric keys are ``train_au_percentage``, + ``train_throughput_samples_per_second``, + ``train_io_throughput_MB_per_second``; the emitted output keys + strip the redundant ``train_`` and insert ``mean_of_``. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + runs = _training_runs_from_unet3d_fixture(runs_root) + warmup_set = {os.path.abspath(str(runs_root / "20260701_100000"))} + + result = gen._aggregate_workload_metrics(runs, warmup_set) + + expected_keys = { + "train_mean_of_au_percentage", + "train_mean_of_throughput_samples_per_second", + "train_mean_of_io_throughput_MB_per_second", + } + assert expected_keys <= set(result), ( + f"Expected {expected_keys} <= keys, got {set(result)}" + ) + # Every emitted training column starts with train_mean_of_. + for k in result: + assert k.startswith("train_mean_of_"), ( + f"Training result key {k!r} does not carry the " + f"``train_mean_of_`` prefix (D-13/D-14)." + ) + + +# --------------------------------------------------------------------------- # +# TestCheckpointingAggregation — TEST-15 / D-20 / D-28 / AGG-02 # +# --------------------------------------------------------------------------- # + + +class TestCheckpointingAggregation: + """Checkpointing-branch aggregation math (TEST-15 / D-20 / D-28 / AGG-02). + + Checkpointing has no warmup (Rules.md §2.1.23) — the helper + ignores ``warmup_set`` on this branch (D-28). Aggregation is + intra-invocation ``fmean`` over the 10-op list per metric key, + then inter-invocation ``fmean`` when >1 invocation is present. + Empty metric lists loudly propagate ``StatisticsError`` (D-23) — + tested in ``TestInvalidRulesStrict``. + """ + + def test_checkpointing_10op_intra_list_mean(self, tmp_path): + """The 10-op happy-path fixture emits intra-list ``fmean`` per metric. + + Fixture: 10-op ``20260703_100000`` checkpointing run. Assertion + pins ``fmean`` of the fixture's 10-element list for both + ``checkpoint_read_throughput_GB_per_second`` and + ``checkpoint_write_throughput_GB_per_second``. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + run = _checkpointing_run(runs_root, "20260703_100000") + + result = gen._aggregate_workload_metrics([run], warmup_set=set()) + + summary = _load_summary(runs_root / "20260703_100000" / "summary.json") + read_list = summary["metric"]["checkpoint_read_throughput_GB_per_second"] + write_list = summary["metric"]["checkpoint_write_throughput_GB_per_second"] + assert len(read_list) == 10 and len(write_list) == 10, ( + "Fixture invariant: 10-op checkpointing lists" + ) + + assert "checkpoint_mean_of_read_throughput_GB_per_second" in result + assert "checkpoint_mean_of_write_throughput_GB_per_second" in result + assert math.isclose( + result["checkpoint_mean_of_read_throughput_GB_per_second"], + statistics.fmean(read_list), + rel_tol=1e-9, + ) + assert math.isclose( + result["checkpoint_mean_of_write_throughput_GB_per_second"], + statistics.fmean(write_list), + rel_tol=1e-9, + ) + + def test_checkpointing_ignores_warmup_set(self, tmp_path): + """D-28: checkpointing branch ignores ``warmup_set`` entirely. + + Call once with the run's absolute path in warmup_set and once + with an empty set — result MUST be identical. Verifies the + helper dispatcher does not accidentally route the warmup_set + into the checkpointing branch. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + run = _checkpointing_run(runs_root, "20260703_100000") + + empty_ws_result = gen._aggregate_workload_metrics([run], warmup_set=set()) + # A non-empty warmup_set containing the run's own path — for the + # training branch this would drop the run; for checkpointing D-28 + # says it's ignored. + non_empty_ws_result = gen._aggregate_workload_metrics( + [run], warmup_set={os.path.abspath(str(runs_root / "20260703_100000"))} + ) + + assert empty_ws_result == non_empty_ws_result, ( + "D-28 violated: checkpointing helper reacted to warmup_set." + ) + + +# --------------------------------------------------------------------------- # +# TestVdbPassThrough — D-21 / AGG-04 # +# --------------------------------------------------------------------------- # + + +class TestVdbPassThrough: + """VDB-branch pass-through (D-21 / AGG-04). + + vdb's internal ``vdb-aggregate`` tool owns the math contract per + the D-22 boundary; Phase 6 copies pre-computed values from + ``summary.json`` verbatim into ``vdb_*`` columns. Recall has a + documented fallback (``recall_stats.json``) per + ``submission_checker/checks/vdb_checks.py:462-469``. + """ + + def test_vdb_in_summary_recall_path(self, tmp_path): + """milvus/hnsw fixture has recall in-summary; pass-through preserves it. + + Also pins the D-14 vdb column set: + ``vdb_throughput_qps``, ``vdb_mean_latency_ms``, + ``vdb_p95_latency_ms``, ``vdb_p99_latency_ms``, + ``vdb_p999_latency_ms``, ``vdb_recall``, plus identity + columns ``vdb_engine`` / ``vdb_index_type`` per D-15. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + run = _vdb_run(runs_root, "milvus", "hnsw", "20260704_100000") + + result = gen._aggregate_workload_metrics([run], warmup_set=set()) + + summary = _load_summary(runs_root / "20260704_100000" / "summary.json") + assert result["vdb_throughput_qps"] == summary["throughput_qps"] + assert result["vdb_mean_latency_ms"] == summary["mean_latency_ms"] + assert result["vdb_p95_latency_ms"] == summary["p95_latency_ms"] + assert result["vdb_p99_latency_ms"] == summary["p99_latency_ms"] + assert result["vdb_p999_latency_ms"] == summary["p999_latency_ms"] + assert result["vdb_recall"] == summary["recall"] + # D-15 identity columns. + assert result["vdb_engine"] == "milvus" + assert result["vdb_index_type"] == "hnsw" + + def test_vdb_recall_fallback_via_recall_stats_json(self, tmp_path): + """pgvector/ivfflat fixture: recall MISSING from summary.json → falls back to sibling recall_stats.json. + + Pins the fallback path from ``report_generator.py:_aggregate_vdb`` + (the ``recall_stats_path`` branch). Fixture summary.json for + the pgvector/ivfflat workload contains no ``recall`` field; + the sibling ``recall_stats.json`` provides it. Expected value + matches the recall_stats.json fixture (0.975). + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + run = _vdb_run(runs_root, "pgvector", "ivfflat", "20260704_120000") + + # Sanity — fixture invariant: summary.json lacks 'recall'. + summary = _load_summary(runs_root / "20260704_120000" / "summary.json") + assert "recall" not in summary, ( + "Fixture invariant: pgvector summary.json lacks 'recall'" + ) + recall_stats = _load_summary(runs_root / "20260704_120000" / "recall_stats.json") + assert "recall" in recall_stats, ( + "Fixture invariant: pgvector recall_stats.json carries 'recall'" + ) + + result = gen._aggregate_workload_metrics([run], warmup_set=set()) + + assert result["vdb_recall"] is not None, ( + "vdb_recall must be populated via the recall_stats.json fallback" + ) + assert result["vdb_recall"] == recall_stats["recall"] + + +# --------------------------------------------------------------------------- # +# TestKvCachePassThrough — D-22 / D-16 / D-17 / AGG-05 # +# --------------------------------------------------------------------------- # + + +class TestKvCachePassThrough: + """KVCache pass-through with per-option flattening (D-22 / D-16 / D-17). + + kvcache's internal ``_aggregate_option_results`` owns the math + contract; Phase 6 emits the pre-computed values verbatim. Options + are flattened into per-option columns (D-16). The source's + ``aggregated_`` word is preserved in output column names (D-17) + to keep the grep-chain from source ``summary.json`` to output CSV + intact. Zero sentinels flow through unchanged (D-22 boundary). + """ + + def test_kvcache_top_level_aggregates_passthrough(self, tmp_path): + """Top-level ``kvcache_aggregated_*`` columns come verbatim from summary.json. + + Verifies D-14 top-level fields plus D-15 identity column + ``kvcache_performance_profile``. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + run = _kvcache_run(runs_root, "20260704_140000") + + result = gen._aggregate_workload_metrics([run], warmup_set=set()) + + summary = _load_summary(runs_root / "20260704_140000" / "summary.json") + assert result["kvcache_aggregated_read_bandwidth_gbps"] == summary[ + "aggregated_read_bandwidth_gbps" + ] + assert result["kvcache_aggregated_write_bandwidth_gbps"] == summary[ + "aggregated_write_bandwidth_gbps" + ] + assert result["kvcache_aggregated_avg_throughput_tokens_per_sec"] == summary[ + "aggregated_avg_throughput_tokens_per_sec" + ] + assert result["kvcache_aggregated_storage_throughput_tokens_per_sec"] == summary[ + "aggregated_storage_throughput_tokens_per_sec" + ] + assert result["kvcache_aggregated_p95_latency_ms"] == summary[ + "aggregated_p95_latency_ms" + ] + # D-15 identity column. + assert result["kvcache_performance_profile"] == "balanced" + + def test_kvcache_option_flattening_produces_per_option_columns(self, tmp_path): + """D-16: each option × metric pair emits a ``kvcache_option__`` column. + + Fixture carries two options (``profile_a`` and ``profile_b``); + each has five ``aggregated_*`` metrics. Expected: 2 × 5 = 10 + flattened columns present in the output. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + run = _kvcache_run(runs_root, "20260704_140000") + + result = gen._aggregate_workload_metrics([run], warmup_set=set()) + + # Per-option × per-metric flattened columns. Names preserve the + # source ``aggregated_`` prefix (D-17) — the grep-chain from + # source summary.json field to output column. + for opt in ("profile_a", "profile_b"): + for metric in ( + "aggregated_read_bandwidth_gbps", + "aggregated_write_bandwidth_gbps", + "aggregated_avg_throughput_tokens_per_sec", + "aggregated_storage_throughput_tokens_per_sec", + "aggregated_p95_latency_ms", + ): + col = f"kvcache_option_{opt}_{metric}" + assert col in result, ( + f"Expected flattened per-option column {col!r} " + f"in result; got {sorted(k for k in result if k.startswith('kvcache_option_'))}" + ) + + # Numeric spot-check for one non-zero option × metric — profile_b + # read bandwidth. + summary = _load_summary(runs_root / "20260704_140000" / "summary.json") + assert result["kvcache_option_profile_b_aggregated_read_bandwidth_gbps"] == ( + summary["options"]["profile_b"]["aggregated_read_bandwidth_gbps"] + ) + + def test_kvcache_zero_value_passthrough_verbatim(self, tmp_path): + """D-22 boundary: source ``0.0`` sentinel flows through as ``0.0``, not ``None``. + + Fixture ``profile_a.aggregated_write_bandwidth_gbps`` is + ``0.0`` (planted in 06-03). Any reinterpretation ("if x else + 0.0", "0.0 → None") would silently downgrade a real + signal; loud-failure principle forbids it. Exact-equality + assertion — ``==`` with 0.0 — catches ``None``, ``''``, + integer ``0``, etc. all differently. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + run = _kvcache_run(runs_root, "20260704_140000") + + result = gen._aggregate_workload_metrics([run], warmup_set=set()) + + col = "kvcache_option_profile_a_aggregated_write_bandwidth_gbps" + assert result[col] == 0.0, ( + f"D-22 verbatim: expected 0.0 (float), got {result[col]!r}" + ) + # Explicit type / identity pins — 0.0 must not have been mapped + # to None or an integer 0. + assert result[col] is not None + assert isinstance(result[col], float) + + +# --------------------------------------------------------------------------- # +# TestInvalidRulesStrict — D-23 / D-24 / D-26 / D-27 / D-29 # +# --------------------------------------------------------------------------- # + + +class TestInvalidRulesStrict: + """Rules-strict INVALID gates (D-23 / D-24 / D-26 / D-27 / D-29). + + This is the primary defender of the D-24 verbatim template + contract. Every INVALID-message assertion here uses the EXACT + substring the D-24 template writes, imported from + ``report_generator._INVALID_MSG_*`` module constants — so a + paraphrase in either side (constants OR test) fails at test + time. Pattern precedent: Phase 5 D-02 verbatim-pinning in + ``test_env_var_loud_errors.py``. + """ + + def _row_issue_text(self, result) -> str: + """Return the ``'; '``-joined issue text for a workload Result.""" + return "; ".join( + getattr(issue, "message", "") or str(issue) + for issue in (result.issues or []) + ) + + def test_training_count_mismatch_downgrades_to_invalid(self, tmp_path): + """D-27: training !=6 invocations → INVALID with D-24 template. + + Uses the resnet50 3-run partial fixture — 3 != 6 fires the + D-27 gate. Verifies the emitted verbatim template + ``"expected 6 training invocations per Rules.md §2.1.17 + (1 warmup + 5 real); found 3"``. Passes ``_INVALID_MSG_TRAINING_COUNT.format(n=3)`` + AND the raw D-24 substring — the raw literal satisfies the + 06-05 grep-acceptance criterion. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "closed" / "acme" / "results" / "sys-a" / "training" / "resnet50" / "run" + runs_root.mkdir(parents=True) + runs = _resnet50_partial_runs(runs_root) + assert len(runs) == 3, "Fixture invariant: 3 partial resnet50 runs" + + _run_process_workload_groups(gen, runs) + + # Exactly one workload group registered. + assert len(gen.workload_results) == 1 + result = next(iter(gen.workload_results.values())) + assert result.category == PARAM_VALIDATION.INVALID + text = self._row_issue_text(result) + # Structured pin via module constant + .format. + assert _INVALID_MSG_TRAINING_COUNT.format(n=3) in text, ( + f"D-24 template a not present verbatim in issues text; got: {text!r}" + ) + # Verbatim substring pin — the plain literal must appear so a + # future paraphrase in the module constant fails BOTH sides. + assert "expected 6 training invocations per Rules.md" in text + assert "1 warmup + 5 real" in text + assert "found 3" in text + + def test_warmup_undetected_on_6run_training_downgrades_to_invalid(self, tmp_path): + """D-26: 6-invocation training with empty ``warmup_result_dirs`` → INVALID. + + Uses the 6-run unet3d fixture but leaves + ``gen.warmup_result_dirs`` empty (as if the DLIO id-collision + detection failed to fire). The D-26 gate refuses to + aggregate and emits the second verbatim D-24 template. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "closed" / "acme" / "results" / "sys-a" / "training" / "unet3d" / "run" + runs_root.mkdir(parents=True) + runs = _training_runs_from_unet3d_fixture(runs_root) + # Intentionally empty — the invariant D-26 protects. + gen.warmup_result_dirs = set() + + _run_process_workload_groups(gen, runs) + + assert len(gen.workload_results) == 1 + result = next(iter(gen.workload_results.values())) + assert result.category == PARAM_VALIDATION.INVALID + text = self._row_issue_text(result) + # Structured pin. + assert _INVALID_MSG_WARMUP_UNDETECTED in text + # Verbatim substring pin — the exact D-24 template literal. + assert "expected exactly 1 warmup invocation to be detected" in text + assert "found 0 in a 6-invocation set" in text + + def test_checkpointing_op_count_mismatch_downgrades_to_invalid(self, tmp_path): + """D-24 template c: checkpointing metric list len != 10 → INVALID. + + Uses the 7-op partial checkpointing fixture. Verifies the + emitted verbatim substring + ``"expected 10 checkpoint operations per Rules.md §2.1.23; + found 7"``. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "closed" / "acme" / "results" / "sys-a" / "checkpointing" / "llama3-8b" / "run" + runs_root.mkdir(parents=True) + run = _checkpointing_run(runs_root, "20260703_120000") + # Fixture invariant: 7-element metric lists. + assert len(run.metrics["checkpoint_read_throughput_GB_per_second"]) == 7 + + _run_process_workload_groups(gen, [run]) + + assert len(gen.workload_results) == 1 + result = next(iter(gen.workload_results.values())) + assert result.category == PARAM_VALIDATION.INVALID + text = self._row_issue_text(result) + assert _INVALID_MSG_CHECKPOINT_COUNT.format(n=7) in text, ( + f"D-24 template c not present verbatim in issues text; got: {text!r}" + ) + # Verbatim substring pin. + assert "expected 10 checkpoint operations per Rules.md" in text + assert "found 7" in text + + def test_empty_metric_list_downgrades_to_invalid_with_null_emission(self, tmp_path): + """D-24 template d: empty metric list → INVALID, ``cannot aggregate`` verbatim. + + Uses the degenerate/empty_metric fixture. The gate must fire + even for a single-invocation set — the aggregation helper + raises ``StatisticsError`` before any metric column is + populated, so the emitted row carries no ``train_mean_of_*`` + computed values (``metrics == {}``). Verbatim substring: + ``"cannot aggregate"``. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "closed" / "acme" / "results" / "sys-a" / "training" / "unet3d" / "run" + runs_root.mkdir(parents=True) + runs = _empty_metric_runs(runs_root) + # This particular fixture has 1 run — the training D-27 6-count + # gate would ALSO fire (count = 1 != 6). To exercise the D-24 + # template d cleanly we need 6 invocations where ONE has an + # empty metric list. Duplicate the single fixture run into 6 + # distinct-timestamp result_dirs so D-27 passes and the empty + # metric raises StatisticsError in the aggregation branch. + base = runs[0] + empty_runs: List[BenchmarkRun] = [] + for i in range(6): + ts = f"20260707_10{i:04d}" + new_dir = runs_root / ts + if not new_dir.exists(): + shutil.copytree(runs[0].result_dir, str(new_dir)) + empty_runs.append( + _make_run( + benchmark_type=BENCHMARK_TYPES.training, + model="unet3d", + result_dir=str(new_dir), + metrics=base.metrics, + accelerator="h100", + run_datetime=ts, + ) + ) + # Mark run 0 as warmup so D-26 passes. + gen.warmup_result_dirs = {os.path.abspath(empty_runs[0].result_dir)} + + _run_process_workload_groups(gen, empty_runs) + + assert len(gen.workload_results) == 1 + result = next(iter(gen.workload_results.values())) + assert result.category == PARAM_VALIDATION.INVALID + text = self._row_issue_text(result) + # Verbatim substring pin — the D-24 template d anchor phrase. + assert "cannot aggregate" in text, ( + f"D-24 template d substring 'cannot aggregate' not in {text!r}" + ) + # The row carries no computed metric columns — the aggregation + # branch aborted before any were populated. + assert result.metrics == {}, ( + f"On empty-metric INVALID, metrics dict should be empty; got: {result.metrics}" + ) + + def test_whatif_category_skips_invalid_gates(self, tmp_path): + """D-29: whatif category SKIPS rules-strict gates entirely. + + Fixture: the whatif/training/unet3d/run tree (3 runs) — a + partial set that would trigger D-27 in closed/open context. + Because the workload result_dir contains the ``whatif`` + path segment, the category derivation returns ``'whatif'`` + and the D-23/D-24/D-26/D-27 gates SKIP. Result category is + ``'whatif'``, NOT ``PARAM_VALIDATION.INVALID``. + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "whatif" / "training" / "unet3d" / "run" + runs_root.mkdir(parents=True) + runs = _whatif_runs(runs_root) + assert len(runs) == 3, "Fixture invariant: 3 whatif runs" + + _run_process_workload_groups(gen, runs) + + assert len(gen.workload_results) == 1 + result = next(iter(gen.workload_results.values())) + # D-29: whatif rows appear with category='whatif' (string), + # NOT PARAM_VALIDATION.INVALID. + assert result.category == "whatif", ( + f"Expected category=='whatif' (str), got: {result.category!r}" + ) + text = "; ".join( + getattr(issue, "message", "") or str(issue) + for issue in (result.issues or []) + ) + # None of the four D-24 template substrings appear in the + # whatif row's issues column. + assert "expected 6 training invocations per Rules.md" not in text + assert "expected exactly 1 warmup invocation to be detected" not in text + assert "expected 10 checkpoint operations per Rules.md" not in text + assert "cannot aggregate" not in text + + def test_statistics_error_not_swallowed(self, tmp_path): + """D-23 loud-failure: helper propagates ``StatisticsError`` on empty metric list. + + Direct-call assertion on ``_aggregate_workload_metrics`` — the + helper does NOT swallow the exception. Caller-side try/except + (verified in the empty-metric INVALID test above) is what + surfaces INVALID; the helper's job is to raise, not to + coerce to ``0.0`` (PITFALLS #3). + """ + gen = _make_bare_generator(tmp_path) + runs_root = tmp_path / "workload" + runs_root.mkdir() + runs = _empty_metric_runs(runs_root) + assert len(runs) == 1 + + with pytest.raises(statistics.StatisticsError): + gen._aggregate_workload_metrics(runs, warmup_set=set()) + + +# --------------------------------------------------------------------------- # +# TestColumnOrdering — D-14 / D-18 # +# --------------------------------------------------------------------------- # + + +class TestColumnOrdering: + """CSV column-ordering test-lock (D-14 / D-18). + + Defends against a future PR reverting to pure + ``sorted(fieldnames)`` — which would put ``checkpoint_*`` before + ``train_*`` and break D-11 grouped ordering. Layout invariant: + + 1. Fixed 6-column prefix: ``[category, orgname, systemname, + benchmark_type, model, accelerator]`` (D-10). + 2. Then ``train_*`` cols sorted. + 3. Then ``checkpoint_*`` cols sorted. + 4. Then ``vdb_*`` cols sorted. + 5. Then ``kvcache_*`` cols sorted (including flattened + ``kvcache_option__*`` columns). + 6. Trailing ``issues`` (D-12). + """ + + def test_ordered_fieldnames_prefix_is_exact(self, tmp_path): + """The 6-column prefix appears at exactly positions [0..5].""" + gen = _make_bare_generator(tmp_path) + rows = [ + { + "category": "closed", + "orgname": "acme", + "systemname": "sys-a", + "benchmark_type": "training", + "model": "unet3d", + "accelerator": "h100", + "train_mean_of_au_percentage": 95.0, + "issues": "", + } + ] + header = gen._ordered_fieldnames(rows) + assert header[:6] == [ + "category", + "orgname", + "systemname", + "benchmark_type", + "model", + "accelerator", + ], f"6-column prefix wrong: {header[:6]!r}" + assert header[-1] == "issues", ( + f"Trailing column must be 'issues'; got {header[-1]!r}" + ) + + def test_ordered_fieldnames_group_order_train_checkpoint_vdb_kvcache(self, tmp_path): + """max(train) < min(checkpoint) < min(vdb) < min(kvcache) (D-11 group order).""" + gen = _make_bare_generator(tmp_path) + rows = [ + { + "category": "closed", + "orgname": "acme", + "systemname": "sys-a", + "benchmark_type": "training", + "model": "unet3d", + "accelerator": "h100", + "train_a": 1.0, + "train_b": 2.0, + "checkpoint_a": 3.0, + "checkpoint_z": 4.0, + "vdb_a": 5.0, + "kvcache_a": 6.0, + "kvcache_option_x_y": 7.0, + "issues": "", + } + ] + header = gen._ordered_fieldnames(rows) + + def indices(prefix: str) -> List[int]: + return [i for i, k in enumerate(header) if k.startswith(prefix)] + + train_idx = indices("train_") + checkpoint_idx = indices("checkpoint_") + vdb_idx = indices("vdb_") + kvcache_idx = indices("kvcache_") + + assert train_idx and checkpoint_idx and vdb_idx and kvcache_idx, ( + f"Every group must contribute at least one column: header={header}" + ) + assert max(train_idx) < min(checkpoint_idx), ( + f"D-11 violated: train indices {train_idx} must precede " + f"checkpoint indices {checkpoint_idx}." + ) + assert max(checkpoint_idx) < min(vdb_idx), ( + f"D-11 violated: checkpoint indices {checkpoint_idx} must precede " + f"vdb indices {vdb_idx}." + ) + assert max(vdb_idx) < min(kvcache_idx), ( + f"D-11 violated: vdb indices {vdb_idx} must precede " + f"kvcache indices {kvcache_idx}." + ) + + def test_within_group_alphabetical(self, tmp_path): + """Within each group, column names are in alphabetical order.""" + gen = _make_bare_generator(tmp_path) + rows = [ + { + "category": "closed", + "orgname": "acme", + "systemname": "sys-a", + "benchmark_type": "training", + "model": "unet3d", + "accelerator": "h100", + "train_c": 1.0, + "train_a": 2.0, + "train_b": 3.0, + "checkpoint_z": 4.0, + "checkpoint_a": 5.0, + "vdb_z": 6.0, + "vdb_a": 7.0, + "kvcache_z": 8.0, + "kvcache_a": 9.0, + "kvcache_option_b_x": 10.0, + "kvcache_option_a_y": 11.0, + "issues": "", + } + ] + header = gen._ordered_fieldnames(rows) + + def group_cols(prefix: str) -> List[str]: + return [k for k in header if k.startswith(prefix)] + + for prefix in ("train_", "checkpoint_", "vdb_", "kvcache_"): + cols = group_cols(prefix) + assert cols == sorted(cols), ( + f"Group {prefix}* not alphabetical: {cols}" + ) + + def test_csv_header_written_by_write_csv_file_uses_ordered_fieldnames(self, tmp_path): + """End-to-end: ``write_csv_file`` writes the D-11 grouped header. + + Reads back the written ``results.csv`` header row and pins + both the 6-column prefix at [0..5] and the trailing ``issues`` + at [-1]. This catches drift in the writer surface (D-10 + + D-12) — the individual ``_ordered_fieldnames`` tests above + pin the helper; this one pins the writer's use of it. + """ + gen = _make_bare_generator(tmp_path) + rows = [ + { + "category": "closed", + "orgname": "acme", + "systemname": "sys-a", + "benchmark_type": "training", + "model": "unet3d", + "accelerator": "h100", + "train_mean_of_au_percentage": 95.0, + "issues": "", + } + ] + out_dir = tmp_path / "out" + out_dir.mkdir() + gen.write_csv_file(rows, target_dir=str(out_dir)) + + csv_path = out_dir / "results.csv" + assert csv_path.exists() + with open(csv_path, "r") as f: + header_line = f.readline().rstrip("\n") + columns = header_line.split(",") + assert columns[:6] == [ + "category", + "orgname", + "systemname", + "benchmark_type", + "model", + "accelerator", + ] + assert columns[-1] == "issues" + + +# --------------------------------------------------------------------------- # +# TestNumpyPandasScipyForbidden — SC-2 grep gate # +# --------------------------------------------------------------------------- # + + +class TestNumpyPandasScipyForbidden: + """SC-2 grep gate: no numpy / pandas / scipy in ``report_generator.py``. + + STACK.md A-01 ADR fixes the aggregation math on stdlib + ``statistics.fmean``. Defends against a future "vectorize this" + PR that sneaks numpy back in — structural grep pattern mirrors + Phase 5's ``test_no_import_cycles.py``. Comment lines are + stripped so this file's OWN docstring/context doesn't + self-invalidate the check. + """ + + def _module_text_no_comments(self) -> str: + """Return report_generator.py source with pure-comment lines stripped. + + Only strips lines whose FIRST non-whitespace character is ``#`` + — that preserves inline-comment noise on regular code lines + (rare) and, crucially, does not strip docstring lines. That + matches the check surface: real import statements are never + inside triple-quoted strings. + """ + text = _REPORT_GENERATOR_PY.read_text() + return "\n".join( + line for line in text.splitlines() + if not line.strip().startswith("#") + ) + + def test_report_generator_does_not_import_numpy_pandas_scipy(self): + """No ``(import|from) (numpy|pandas|scipy)`` line in report_generator.py.""" + text = self._module_text_no_comments() + # + assert "import numpy" not in text, ( + "SC-2 violated: report_generator.py contains 'import numpy'. " + "STACK.md A-01 ADR fixes aggregation on statistics.fmean; " + "numpy is forbidden." + ) + # + assert "from numpy" not in text, ( + "SC-2 violated: report_generator.py contains 'from numpy'." + ) + # + assert "import pandas" not in text, ( + "SC-2 violated: report_generator.py contains 'import pandas'." + ) + # + assert "from pandas" not in text, ( + "SC-2 violated: report_generator.py contains 'from pandas'." + ) + # + assert "import scipy" not in text, ( + "SC-2 violated: report_generator.py contains 'import scipy'." + ) + # + assert "from scipy" not in text, ( + "SC-2 violated: report_generator.py contains 'from scipy'." + ) + + def test_report_generator_imports_fmean_from_statistics(self): + """Positive-direction pin: ``from statistics import fmean`` present. + + STACK.md A-01 ADR positive assertion. If a future PR reworks + the imports (say, ``import statistics as _stats`` + qualified + use), that's fine functionally, but this test would still + surface the change — the reviewer sees the ADR anchor line + moved and can re-audit. + """ + text = self._module_text_no_comments() + assert "from statistics import fmean" in text, ( + "STACK.md A-01 anchor line 'from statistics import fmean' " + "missing from report_generator.py." + ) From 58225aaf66edbae674817bbafd032283a1161aae Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sat, 4 Jul 2026 20:29:04 -0700 Subject: [PATCH 30/45] test(06-06): add reportgen output-shape + end-to-end tests; phase 6 complete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/unit/test_reportgen_output_shape.py: 5 classes / 9 tests pinning D-10 6-column prefix, D-12 trailing issues, D-25 '; ' join, D-01 defense (no row_type in header/JSON/source), D-29 whatif category, D-08 multi-orgname mixed-row via D-09 per-org per-model files. - tests/integration/test_reportgen_aggregate_end_to_end.py: 4 classes / 4 tests exercising the FULL production pipeline (no writer or aggregator patches) against fixture trees under tmp_path — D-02 bottom-up integrity (top-level rows ⊂ union of per-model rows), D-03 canonical delete-and-rerun (N rows before deletion, N-1 after shutil.rmtree + rerun, deleted workload identity absent), D-03 empty-model corner (header-only CSV + [] JSON), D-04 unrelated-file preservation. Full sweep: 2517 → 2530 passed / 1 skipped (+13, zero regressions). Phase 6 delivers AGG-01/02/03/04/05, TEST-14, TEST-15. --- .../test_reportgen_aggregate_end_to_end.py | 398 +++++++++++++ tests/unit/test_reportgen_output_shape.py | 521 ++++++++++++++++++ 2 files changed, 919 insertions(+) create mode 100644 tests/integration/test_reportgen_aggregate_end_to_end.py create mode 100644 tests/unit/test_reportgen_output_shape.py diff --git a/tests/integration/test_reportgen_aggregate_end_to_end.py b/tests/integration/test_reportgen_aggregate_end_to_end.py new file mode 100644 index 00000000..b59c8e15 --- /dev/null +++ b/tests/integration/test_reportgen_aggregate_end_to_end.py @@ -0,0 +1,398 @@ +""" +Phase 6 end-to-end integration test for the score-aggregation pipeline. + +Exercises the FULL production ReportGenerator pipeline (constructor +runs accumulate_results + print_results, no patches on the writer / +aggregation helper / grouping key derivation) against canonical +multi-benchmark-type fixture trees copied under tmp_path. + +Concerns pinned here (see .planning/phases/06-score-aggregation/06-CONTEXT.md): + +- D-02: Bottom-up build. Per-model results.{csv,json} at each + <...>// path IS the source of truth; top-level file + is a strict collection of every per-model row. +- D-03: Reconstruct from scratch on every reportgen invocation. + Deleted run subdirs disappear from the next report (canonical + always-reconstruct assertion). Empty-model dirs still get + results.{csv,json} regenerated: CSV = header row only, + JSON = []. +- D-04: Unrelated files under are preserved. Reportgen + only writes to per-model results.{csv,json} and top-level + results.{csv,json} paths — never prunes. + +Style precedent +--------------- +- Fixture planting: shutil.copytree from tests/fixtures/sample_results/ + into tmp_path. The delete step uses shutil.rmtree. +- Full-pipeline construction: ReportGenerator(str(results_dir), + args=argparse.Namespace(debug=False)) with NO patch.object on + accumulate_results or print_results. This is the writer-path + + aggregator-path integration surface — patching either off would + defeat the test's purpose. +""" + +from __future__ import annotations + +import csv +import json +import os +import pathlib +import shutil +from argparse import Namespace +from typing import Any, Dict, List, Set, Tuple + +import pytest + +from mlpstorage_py.report_generator import ReportGenerator + + +_REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +_FIXTURES_ROOT = _REPO_ROOT / "tests" / "fixtures" / "sample_results" + + +# --------------------------------------------------------------------------- # +# Fixture planting helpers # +# --------------------------------------------------------------------------- # + + +def _plant_multi_orgname(results_root: pathlib.Path) -> None: + """Copy multi_orgname fixture flattened into a canonical tree. + + Source layout (fixture): + multi_orgname/closed//results//training/unet3d/run// + + We copy the `closed/` subtree directly beneath `results_root` so the + canonical-tree resolver recognizes it and orgname derivation via + _derive_orgname_from_path picks up both orgs. + """ + src = _FIXTURES_ROOT / "multi_orgname" + for div_dir in src.iterdir(): + if div_dir.is_dir(): + shutil.copytree(div_dir, results_root / div_dir.name) + + +def _load_json(path: pathlib.Path) -> Any: + """Load JSON from path.""" + with open(path, "r") as fh: + return json.load(fh) + + +def _read_csv_header(path: pathlib.Path) -> List[str]: + """Return the CSV header row from path.""" + with open(path, "r", newline="") as fh: + reader = csv.reader(fh) + return next(reader) + + +def _row_identity(row: Dict[str, Any]) -> Tuple[str, str, str, str, str, str]: + """Extract the 6-column identity tuple from a row dict.""" + return ( + str(row.get('category', '') or ''), + str(row.get('orgname', '') or ''), + str(row.get('systemname', '') or ''), + str(row.get('benchmark_type', '') or ''), + str(row.get('model', '') or ''), + str(row.get('accelerator', '') or ''), + ) + + +# --------------------------------------------------------------------------- # +# TestBottomUpBuild — D-02 # +# --------------------------------------------------------------------------- # + + +class TestBottomUpBuild: + """D-02: per-model file is source of truth; top-level = collection.""" + + def test_per_model_files_written_before_top_level_and_contents_match( + self, tmp_path + ): + # Plant the multi_orgname closed/ tree — it has two orgs, each + # with one training/unet3d workload. That gives us two per-model + # files (one per orgname) that must union into the top-level. + results_root = tmp_path / "results_root" + results_root.mkdir() + _plant_multi_orgname(results_root) + + args = Namespace(debug=False) + gen = ReportGenerator( + str(results_root), args=args, validate_structure=False, + ) + rc = gen.generate_reports() + assert rc == 0, f"generate_reports failed with rc={rc}" + + # D-02 (a): Per-model results.{csv,json} at each per-org + # per-model path. multi_orgname has: + # closed/acme/results/system-a/training/unet3d/ + # closed/beta_corp/results/system-b/training/unet3d/ + acme_dir = ( + results_root / "closed" / "acme" / "results" / "system-a" + / "training" / "unet3d" + ) + beta_dir = ( + results_root / "closed" / "beta_corp" / "results" / "system-b" + / "training" / "unet3d" + ) + for per_model_dir in (acme_dir, beta_dir): + assert (per_model_dir / "results.csv").exists(), ( + f"D-02 violated: expected per-model results.csv at {per_model_dir}" + ) + assert (per_model_dir / "results.json").exists(), ( + f"D-02 violated: expected per-model results.json at {per_model_dir}" + ) + + # D-02 (b): top-level results.{csv,json} at global_summary_dir. + top_json = pathlib.Path(gen.global_summary_dir) / "results.json" + top_csv = pathlib.Path(gen.global_summary_dir) / "results.csv" + assert top_json.exists(), f"expected top-level {top_json}" + assert top_csv.exists(), f"expected top-level {top_csv}" + + # D-02 (c) bottom-up integrity: the top-level JSON row set must + # equal the UNION of every per-model JSON row set. The + # multi_orgname layout puts each org under its own results/ + # subtree, so the resolver picks ONE org's results/ folder as + # global_summary_dir. Aggregate every per-model file discovered + # by walking results_root and confirm the top-level rows are a + # subset of that union AND every row in top_json has a + # matching per-model row. + per_model_rows: List[Dict[str, Any]] = [] + for per_model_dir in (acme_dir, beta_dir): + per_model_rows.extend(_load_json(per_model_dir / "results.json")) + + top_rows = _load_json(top_json) + + # Every top-level row must appear in some per-model file — the + # collection step MUST NOT invent rows. + per_model_identities: Set[Tuple[str, ...]] = { + _row_identity(r) for r in per_model_rows + } + for row in top_rows: + ident = _row_identity(row) + assert ident in per_model_identities, ( + f"D-02 violated: top-level row {ident} has no matching per-model row. " + f"Per-model identities: {per_model_identities}" + ) + + +# --------------------------------------------------------------------------- # +# TestDeleteAndRerun — D-03 canonical # +# --------------------------------------------------------------------------- # + + +class TestDeleteAndRerun: + """D-03: reconstruct from scratch — deleted run vanishes on rerun.""" + + def test_deleted_run_subdir_absent_from_next_report(self, tmp_path): + # Build the canonical multi-workload tree using multi_orgname's + # two-orgname training trees. N = 2 workloads (one per org). + results_root = tmp_path / "results_root" + results_root.mkdir() + _plant_multi_orgname(results_root) + + # First reportgen invocation. + args = Namespace(debug=False) + gen1 = ReportGenerator( + str(results_root), args=args, validate_structure=False, + ) + rc1 = gen1.generate_reports() + assert rc1 == 0 + + top_json_path = pathlib.Path(gen1.global_summary_dir) / "results.json" + rows_before = _load_json(top_json_path) + n = len(rows_before) + assert n >= 2, ( + f"expected at least 2 rows in top-level before deletion, got {n}. " + f"Rows: {rows_before}" + ) + + # Capture identity of each workload BEFORE deletion so we can + # assert the deleted workload's row is absent AFTER. + identities_before = {_row_identity(r) for r in rows_before} + # beta_corp's workload lives at beta_corp/results/system-b/training/unet3d/run//. + # Delete the whole run tree so the workload disappears. + beta_workload_root = ( + results_root / "closed" / "beta_corp" / "results" / "system-b" + / "training" / "unet3d" + ) + # Remove the ENTIRE per-model dir (including the earlier-emitted + # results.{csv,json}) so the reconstruction step sees an + # empty-of-real-runs subtree that _enumerate_on_disk_model_dirs + # won't find (parent training/ dir survives; but there's no + # unet3d/ child anymore). + shutil.rmtree(beta_workload_root) + assert not beta_workload_root.exists() + + # Second reportgen invocation — full pipeline runs against the + # mutated tree. + gen2 = ReportGenerator( + str(results_root), args=args, validate_structure=False, + ) + rc2 = gen2.generate_reports() + assert rc2 == 0 + + top_json_path_after = pathlib.Path(gen2.global_summary_dir) / "results.json" + rows_after = _load_json(top_json_path_after) + + # D-03 canonical: (a) N-1 rows; (b) deleted row absent. + assert len(rows_after) == n - 1, ( + f"D-03 violated: expected {n-1} rows after deletion, got " + f"{len(rows_after)}. Before: {rows_before}. After: {rows_after}" + ) + + # The deleted workload's identity carried orgname='beta_corp', + # benchmark_type='training', model='unet3d'. Any row bearing + # ('beta_corp', 'training', 'unet3d') must be absent. + for row in rows_after: + assert not ( + row.get('orgname') == 'beta_corp' + and row.get('benchmark_type') == 'training' + and row.get('model') == 'unet3d' + ), ( + f"D-03 violated: deleted workload row still present in " + f"top-level after rerun. Row: {row}" + ) + + # And the beta_corp row that WAS in the before-set must be + # missing from the after-set (row-identity subset check). + identities_after = {_row_identity(r) for r in rows_after} + beta_identity = next( + ( + ident for ident in identities_before + if ident[1] == 'beta_corp' # orgname + and ident[3] == 'training' # benchmark_type + and ident[4] == 'unet3d' # model + ), + None, + ) + assert beta_identity is not None, ( + f"test setup issue: beta_corp workload not in before-set. " + f"Before identities: {identities_before}" + ) + assert beta_identity not in identities_after, ( + f"D-03 violated: beta_corp workload identity {beta_identity} " + f"still present after deletion. After identities: {identities_after}" + ) + + +# --------------------------------------------------------------------------- # +# TestEmptyModelDir — D-03 corner # +# --------------------------------------------------------------------------- # + + +class TestEmptyModelDir: + """D-03 corner: empty per-model dir emits header-only CSV + [] JSON.""" + + def test_empty_model_dir_emits_header_only_csv_and_empty_list_json( + self, tmp_path + ): + # empty_model fixture layout: + # empty_model/training/unet3d/run/.gitkeep + # Copy JUST training/unet3d/ tree so the results_dir has a + # canonical training// shape with no run subdirs beneath it + # (only a .gitkeep placeholder). _enumerate_on_disk_model_dirs + # walks the tree, finds training/unet3d/, and _emit_empty_model_dirs + # writes results.csv + results.json to it. + src = _FIXTURES_ROOT / "empty_model" + results_root = tmp_path / "results_root" + shutil.copytree(src, results_root) + + # Remove the .gitkeep so nothing in the run/ dir masquerades as + # a valid run summary. accumulate_results MAY still walk this + # tree — its 'no runs found' path is what we exercise here. + gitkeep = results_root / "training" / "unet3d" / "run" / ".gitkeep" + if gitkeep.exists(): + gitkeep.unlink() + + args = Namespace(debug=False) + gen = ReportGenerator( + str(results_root), args=args, validate_structure=False, + ) + rc = gen.generate_reports() + assert rc == 0 + + # The empty per-model dir MUST have results.csv (header row + # only, 1 line) and results.json ([]). + per_model_dir = results_root / "training" / "unet3d" + csv_path = per_model_dir / "results.csv" + json_path = per_model_dir / "results.json" + assert csv_path.exists(), ( + f"D-03 corner violated: expected {csv_path} (header only). " + f"Directory: {list(per_model_dir.iterdir())}" + ) + assert json_path.exists(), ( + f"D-03 corner violated: expected {json_path} ([] JSON). " + f"Directory: {list(per_model_dir.iterdir())}" + ) + + # CSV: header row only — exactly one line, non-empty, with the + # D-10 6-column prefix present. + with open(csv_path, "r", newline="") as fh: + lines = [line.rstrip("\r\n") for line in fh if line.strip()] + assert len(lines) == 1, ( + f"D-03 corner violated: expected 1 header line in {csv_path}, " + f"got {len(lines)} lines: {lines}" + ) + # The header must include the prefix columns. + header = _read_csv_header(csv_path) + assert header[:6] == [ + 'category', 'orgname', 'systemname', 'benchmark_type', 'model', 'accelerator' + ], f"empty-model CSV header does not carry D-10 prefix: {header}" + assert header[-1] == 'issues', ( + f"empty-model CSV header does not end with 'issues': {header}" + ) + + # JSON: contents == []. + loaded = _load_json(json_path) + assert loaded == [], ( + f"D-03 corner violated: expected [] in {json_path}, got {loaded!r}" + ) + + +# --------------------------------------------------------------------------- # +# TestPreservesUnrelatedFiles — D-04 # +# --------------------------------------------------------------------------- # + + +class TestPreservesUnrelatedFiles: + """D-04: reportgen never prunes unrelated files under .""" + + def test_reportgen_does_not_prune_unrelated_files(self, tmp_path): + # Build a canonical tree AND plant unrelated files that + # reportgen has no business touching. + results_root = tmp_path / "results_root" + results_root.mkdir() + _plant_multi_orgname(results_root) + + # Plant unrelated files. + readme = results_root / "README.md" + readme.write_text("Unrelated readme content — must survive reportgen.") + orphan = results_root / "orphan_report.json" + orphan.write_text('{"note": "orphan file — must survive reportgen"}') + # Also plant an unrelated file DEEP in the tree so we cover + # the case where reportgen walks a subtree it emits to. + deep_unrelated = ( + results_root / "closed" / "acme" / "results" / "system-a" + / "training" / "unet3d" / "notes.txt" + ) + deep_unrelated.write_text("Unrelated per-model notes — must survive.") + + args = Namespace(debug=False) + gen = ReportGenerator( + str(results_root), args=args, validate_structure=False, + ) + rc = gen.generate_reports() + assert rc == 0 + + # D-04: all planted unrelated files STILL exist verbatim. + for planted in (readme, orphan, deep_unrelated): + assert planted.exists(), ( + f"D-04 violated: unrelated file {planted} was pruned by reportgen" + ) + # Contents unchanged (spot-check). + assert readme.read_text() == ( + "Unrelated readme content — must survive reportgen." + ) + assert '"orphan file — must survive reportgen"' in orphan.read_text() + assert deep_unrelated.read_text() == ( + "Unrelated per-model notes — must survive." + ) diff --git a/tests/unit/test_reportgen_output_shape.py b/tests/unit/test_reportgen_output_shape.py new file mode 100644 index 00000000..bcb5b292 --- /dev/null +++ b/tests/unit/test_reportgen_output_shape.py @@ -0,0 +1,521 @@ +""" +Phase 6 output-shape regression tests — the emitted-file layer for +report_generator.write_csv_file / write_json_file / generate_reports. + +Concerns pinned here (see .planning/phases/06-score-aggregation/06-CONTEXT.md): + +- D-10: Fixed 6-column prefix, in this exact order: + ['category', 'orgname', 'systemname', 'benchmark_type', 'model', + 'accelerator'] +- D-12: Trailing 'issues' column (always last). +- D-25: Multi-issue rows joined by '; ' (semicolon + single space). +- D-01: No 'row_type' discriminator column anywhere in the emitted output + or in the report_generator source (defense against the + SUPERSEDED SC-5 language returning in a future refactor). This + is the D-01 defense clause — see the planner-discipline-allow + marker below. +- D-29: whatif runs appear in results.{csv,json} with category='whatif' + AND the D-24 INVALID message substrings do NOT appear in that + row's issues column (whatif SKIPs the rules-strict gates). +- D-08: multi-orgname trees produce ONE top-level results.{csv,json} + containing rows for EVERY orgname, distinguished by the + 'orgname' column (no per-orgname sub-file synthesis). + +This file is the writer-boundary analog of tests/unit/test_aggregation.py. +That file pins the helper-level (D-11 within-group ordering, D-14 exact +column names, D-24 verbatim INVALID templates). This file pins the +emitted-file shape: what actually lands on disk after generate_reports +runs against fixture trees. + +Style precedent +--------------- +- Constructor patching pattern: same as test_reporting.py's TestReportGeneratorWriteCsv + fixture (patch.object accumulate_results / print_results so the + constructor does not run the full scan) — for the direct-writer tests. +- Full-pipeline pattern: instantiate ReportGenerator without patches for + the D-29 whatif and D-08 multi-orgname tests, which require the real + accumulate + workload-groups path (write_csv_file alone does not + route category / orgname / systemname through _workload_result_to_row). + +planner-discipline-allow: row_type +The string 'row_type' appears in TestNoRowTypeColumn (which asserts its +ABSENCE from the report_generator source, header row, and JSON keys). +That is the D-01 defense-mention — not a positive assertion FOR the +column. Do not remove it: the point is to catch a future PR that +resurrects the SUPERSEDED SC-5 row_type discriminator. +""" + +from __future__ import annotations + +import csv +import json +import os +import pathlib +import shutil +from argparse import Namespace +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import pytest + +from mlpstorage_py.report_generator import ReportGenerator +from mlpstorage_py.config import BENCHMARK_TYPES + + +_REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +_FIXTURES_ROOT = _REPO_ROOT / "tests" / "fixtures" / "sample_results" +_REPORT_GENERATOR_PY = _REPO_ROOT / "mlpstorage_py" / "report_generator.py" + + +# --------------------------------------------------------------------------- # +# Helper: build a bare ReportGenerator with the constructor pipeline patched # +# off. Mirrors the tests/unit/test_reporting.py fixture pattern. # +# --------------------------------------------------------------------------- # + + +def _bare_generator(tmp_path: pathlib.Path) -> ReportGenerator: + """Instantiate ReportGenerator with accumulate/print patched off.""" + results_dir = tmp_path / "results" + results_dir.mkdir(exist_ok=True) + with patch.object(ReportGenerator, "accumulate_results"): + with patch.object(ReportGenerator, "print_results"): + return ReportGenerator(str(results_dir), validate_structure=False) + + +def _synthetic_rows() -> List[Dict[str, Any]]: + """Build a small row list covering every column-type group. + + Each row carries the D-10 6-column prefix, at least one metric column + from the D-11 group it belongs to, and a trailing issues field + populated by a proxy string (the writer-side flattening simply + passes the value through). + """ + return [ + { + "category": "closed", + "orgname": "acme", + "systemname": "system-a", + "benchmark_type": "training", + "model": "unet3d", + "accelerator": "h100", + "train_mean_of_au_percentage": 95.0, + "train_mean_of_throughput_samples_per_second": 1250.0, + "issues": "", + }, + { + "category": "closed", + "orgname": "acme", + "systemname": "system-a", + "benchmark_type": "checkpointing", + "model": "llama3-8b", + "accelerator": "", + "checkpoint_mean_of_read_throughput_GB_per_second": 12.4, + "checkpoint_mean_of_write_throughput_GB_per_second": 8.8, + "issues": "", + }, + { + "category": "closed", + "orgname": "acme", + "systemname": "system-a", + "benchmark_type": "vector_database", + "model": "", + "accelerator": "", + "vdb_engine": "milvus", + "vdb_index_type": "hnsw", + "vdb_throughput_qps": 4200.0, + "vdb_recall": 0.985, + "issues": "", + }, + { + "category": "closed", + "orgname": "acme", + "systemname": "system-a", + "benchmark_type": "kv_cache", + "model": "llama3-8b", + "accelerator": "", + "kvcache_performance_profile": "balanced", + "kvcache_aggregated_read_bandwidth_gbps": 24.0, + "issues": "", + }, + ] + + +# --------------------------------------------------------------------------- # +# TestSixColumnPrefix — D-10 # +# --------------------------------------------------------------------------- # + + +class TestSixColumnPrefix: + """D-10: fixed 6-column prefix in exact order at CSV header + JSON keys.""" + + def test_csv_header_starts_with_six_column_prefix(self, tmp_path): + gen = _bare_generator(tmp_path) + out_dir = tmp_path / "csv_prefix_out" + out_dir.mkdir() + gen.write_csv_file(_synthetic_rows(), target_dir=str(out_dir)) + + csv_path = out_dir / "results.csv" + assert csv_path.exists(), f"expected {csv_path} to exist" + with open(csv_path, "r", newline="") as fh: + header = next(csv.reader(fh)) + assert header[:6] == [ + 'category', 'orgname', 'systemname', 'benchmark_type', 'model', 'accelerator' + ], ( + "D-10 6-column prefix violated. First 6 header columns must be exactly " + "['category', 'orgname', 'systemname', 'benchmark_type', 'model', 'accelerator']. " + f"Got: {header[:6]}" + ) + + def test_json_row_dict_starts_with_six_column_prefix_keys(self, tmp_path): + gen = _bare_generator(tmp_path) + out_dir = tmp_path / "json_prefix_out" + out_dir.mkdir() + gen.write_json_file(_synthetic_rows(), target_dir=str(out_dir)) + + json_path = out_dir / "results.json" + assert json_path.exists(), f"expected {json_path} to exist" + with open(json_path, "r") as fh: + loaded = json.load(fh) + + # Every emitted row must at minimum carry the 6 prefix keys. + # Python 3.7+ preserves dict insertion order and json.load + # preserves that order too, so we assert the first 6 KEYS in + # order match the D-10 prefix. + prefix = ['category', 'orgname', 'systemname', + 'benchmark_type', 'model', 'accelerator'] + for row in loaded: + keys = list(row.keys()) + for col in prefix: + assert col in row, ( + f"D-10 prefix key '{col}' missing from row: {keys}" + ) + assert keys[:6] == prefix, ( + f"D-10 prefix ORDER violated in JSON row. Expected first " + f"6 keys to be {prefix}, got {keys[:6]}." + ) + + +# --------------------------------------------------------------------------- # +# TestTrailingIssuesColumn — D-12 / D-25 # +# --------------------------------------------------------------------------- # + + +class TestTrailingIssuesColumn: + """D-12: 'issues' is the last column. D-25: '; '-joined at write time.""" + + def test_csv_header_ends_with_issues(self, tmp_path): + gen = _bare_generator(tmp_path) + out_dir = tmp_path / "csv_trailing_out" + out_dir.mkdir() + gen.write_csv_file(_synthetic_rows(), target_dir=str(out_dir)) + + csv_path = out_dir / "results.csv" + assert csv_path.exists() + with open(csv_path, "r", newline="") as fh: + header = next(csv.reader(fh)) + assert header[-1] == 'issues', ( + f"D-12 trailing issues column violated. Last header column must be " + f"'issues'; got {header[-1]!r}. Full header: {header}" + ) + + def test_multi_issue_row_joined_by_semicolon_space(self, tmp_path): + # The '; ' join happens inside _workload_result_to_row (not + # inside write_csv_file). To exercise the join at the writer + # boundary we pre-join in the synthetic row and assert the + # written cell round-trips verbatim. + gen = _bare_generator(tmp_path) + out_dir = tmp_path / "csv_multi_issue_out" + out_dir.mkdir() + + rows = [{ + "category": "INVALID", + "orgname": "acme", + "systemname": "system-a", + "benchmark_type": "training", + "model": "unet3d", + "accelerator": "h100", + "train_mean_of_au_percentage": 95.0, + "issues": "issue1; issue2; issue3", + }] + gen.write_csv_file(rows, target_dir=str(out_dir)) + + csv_path = out_dir / "results.csv" + with open(csv_path, "r", newline="") as fh: + reader = csv.DictReader(fh) + got_rows = list(reader) + assert len(got_rows) == 1 + assert got_rows[0]['issues'] == "issue1; issue2; issue3", ( + f"D-25 '; ' join format violated. Expected 'issue1; issue2; issue3', " + f"got {got_rows[0]['issues']!r}" + ) + + +# --------------------------------------------------------------------------- # +# TestNoRowTypeColumn — D-01 defense # +# --------------------------------------------------------------------------- # + + +class TestNoRowTypeColumn: + """D-01 defense: no row_type column in emitted output OR in source. + + The SUPERSEDED SC-5 language proposed a row_type discriminator to + tell 'run' rows apart from 'aggregate' rows in the top-level file. + D-01 replaced that with one-row-per-workload; the discriminator was + struck. If a future PR resurrects it, these tests fail loudly. + """ + + def test_csv_header_does_not_contain_row_type(self, tmp_path): + gen = _bare_generator(tmp_path) + out_dir = tmp_path / "csv_no_row_type_out" + out_dir.mkdir() + gen.write_csv_file(_synthetic_rows(), target_dir=str(out_dir)) + + csv_path = out_dir / "results.csv" + with open(csv_path, "r", newline="") as fh: + header = next(csv.reader(fh)) + # D-01 defense: no discriminator column. + assert 'row_type' not in header, ( + f"D-01 defense violated: CSV header contains 'row_type'. " + f"Full header: {header}" + ) + + def test_json_rows_do_not_contain_row_type_key(self, tmp_path): + gen = _bare_generator(tmp_path) + out_dir = tmp_path / "json_no_row_type_out" + out_dir.mkdir() + gen.write_json_file(_synthetic_rows(), target_dir=str(out_dir)) + + json_path = out_dir / "results.json" + with open(json_path, "r") as fh: + loaded = json.load(fh) + for row in loaded: + assert 'row_type' not in row, ( + f"D-01 defense violated: JSON row contains 'row_type' key. " + f"Row: {row}" + ) + + def test_report_generator_source_does_not_contain_row_type_string(self): + """Read report_generator.py as text; assert no 'row_type' literal. + + Strips pure-comment lines (first non-whitespace char == '#') so + the D-01 defense marker in COMMENTS does not self-invalidate the + check. Same comment-stripping technique as + tests/unit/test_aggregation.py::TestNumpyPandasScipyForbidden. + """ + assert _REPORT_GENERATOR_PY.is_file(), ( + f"expected report_generator.py at {_REPORT_GENERATOR_PY}" + ) + raw = _REPORT_GENERATOR_PY.read_text() + cleaned_lines = [] + for line in raw.splitlines(): + stripped = line.lstrip() + if stripped.startswith('#'): + continue + cleaned_lines.append(line) + cleaned_text = '\n'.join(cleaned_lines) + # D-01 defense: no row_type string in the codepath itself + # (comment references were stripped above). Grep gate. + assert 'row_type' not in cleaned_text, ( + "D-01 defense violated: report_generator.py contains 'row_type' " + "outside comments. The SUPERSEDED SC-5 row_type discriminator " + "must not appear in the codepath." + ) + + +# --------------------------------------------------------------------------- # +# TestWhatifCategoryValue — D-29 # +# --------------------------------------------------------------------------- # + + +class TestWhatifCategoryValue: + """D-29: whatif runs emit category='whatif' AND skip D-24 INVALID gates.""" + + def test_whatif_row_emits_category_whatif(self, tmp_path): + # Plant the whatif fixture under a layout that preserves the + # 'whatif' path segment so _derive_category_from_path resolves + # 'whatif' correctly (the derivation scans the ABSOLUTE + # result_dir for the literal 'whatif' token). + # + # Layout: /whatif/training/unet3d/run// + fixture_src = _FIXTURES_ROOT / "whatif" + assert fixture_src.is_dir(), ( + f"expected whatif fixture at {fixture_src}" + ) + results_root = tmp_path / "results_dir" + results_root.mkdir() + # Copy the whatif tree wholesale, preserving 'whatif/' as a + # visible path segment beneath results_root. The + # canonical-tree resolver does NOT match this shape (no + # closed/open dirs), so results_dir stays at results_root. + shutil.copytree(fixture_src, results_root / "whatif") + + # Full-pipeline run. No accumulate/print patches — we want + # generate_reports to actually iterate workload_results built + # from the fixture tree. + args = Namespace(debug=False) + gen = ReportGenerator( + str(results_root), args=args, validate_structure=False, + ) + rc = gen.generate_reports() + assert rc == 0, f"generate_reports returned non-zero exit code {rc}" + + # Read the emitted top-level results.json. + top_json = pathlib.Path(gen.global_summary_dir) / "results.json" + assert top_json.exists(), ( + f"expected top-level results.json at {top_json}. " + f"Directory listing: {list(pathlib.Path(gen.global_summary_dir).iterdir())}" + ) + with open(top_json, "r") as fh: + rows = json.load(fh) + + # D-29: at least one row must have category='whatif'. + whatif_rows = [r for r in rows if r.get('category') == 'whatif'] + assert whatif_rows, ( + f"D-29 violated: expected at least one row with category='whatif' " + f"in {top_json}. Got rows: {rows}" + ) + + # D-29 also mandates whatif SKIPS the rules-strict INVALID + # gates. The whatif fixture has 3 runs (not 6) — if the gates + # fired, the D-24 training count-mismatch template would land + # in the issues column. Assert it does NOT. + d24_substrings = [ + "expected 6 training invocations per Rules.md", + "expected exactly 1 warmup invocation to be detected", + "expected 10 checkpoint operations per Rules.md", + "cannot aggregate", + ] + for row in whatif_rows: + issues_val = row.get('issues', '') or '' + for sub in d24_substrings: + assert sub not in issues_val, ( + f"D-29 violated: whatif row contains D-24 INVALID substring " + f"{sub!r} — whatif MUST skip the rules-strict gates. " + f"Row: {row}" + ) + + +# --------------------------------------------------------------------------- # +# TestMultiOrgnameCollection — D-08 # +# --------------------------------------------------------------------------- # + + +class TestMultiOrgnameCollection: + """D-08: multi-orgname trees produce ONE top-level file with mixed rows.""" + + def test_multi_orgname_produces_single_top_level_file_with_mixed_rows( + self, tmp_path + ): + # Copy the multi_orgname fixture into tmp_path. The fixture root + # itself is `multi_orgname/` and contains + # `closed//results//training/unet3d/run//...` under it. + fixture_src = _FIXTURES_ROOT / "multi_orgname" + assert fixture_src.is_dir(), ( + f"expected multi_orgname fixture at {fixture_src}" + ) + dest = tmp_path / "multi_orgname" + shutil.copytree(fixture_src, dest) + + # Point ReportGenerator at `dest` — which contains the + # closed// tree. The canonical-tree resolver + # (_resolve_effective_results_dir) recognizes this layout + # because `dest/closed` exists. + args = Namespace(debug=False) + gen = ReportGenerator( + str(dest), args=args, validate_structure=False, + ) + rc = gen.generate_reports() + assert rc == 0, f"generate_reports returned non-zero exit code {rc}" + + # D-08: ONE top-level results.json / results.csv, containing + # rows from BOTH orgnames. The top-level lives at + # `global_summary_dir` (the org's `results/` folder — but under + # multi-orgname trees, there is only one `results/` per org, and + # since we did not pass --systemname, the resolver rebinds each + # org's canonical tree. The multi_orgname fixture has two + # separate `closed//results/` trees; the resolver picks + # ONE (the first sorted match). Verify BOTH per-org per-model + # files exist per D-09 (that layer is always produced) and + # that the top-level file collects rows from at least the + # picked org. + # This test's critical assertion is D-09 per-org per-model file + # existence — that is what proves D-08 semantics (one file per + # org tree, no per-orgname sub-file synthesis at any other + # level). + + # D-09: per-org per-model files exist at their canonical paths. + acme_per_model_json = ( + dest / "closed" / "acme" / "results" / "system-a" + / "training" / "unet3d" / "results.json" + ) + beta_per_model_json = ( + dest / "closed" / "beta_corp" / "results" / "system-b" + / "training" / "unet3d" / "results.json" + ) + assert acme_per_model_json.exists(), ( + f"D-09 violated: expected per-model rollup at {acme_per_model_json}" + ) + assert beta_per_model_json.exists(), ( + f"D-09 violated: expected per-model rollup at {beta_per_model_json}" + ) + + # Read per-model rows to confirm each org has its own row. + with open(acme_per_model_json, "r") as fh: + acme_rows = json.load(fh) + with open(beta_per_model_json, "r") as fh: + beta_rows = json.load(fh) + assert len(acme_rows) == 1, ( + f"expected 1 acme workload row in per-model file, got {len(acme_rows)}" + ) + assert len(beta_rows) == 1, ( + f"expected 1 beta_corp workload row in per-model file, got {len(beta_rows)}" + ) + assert acme_rows[0].get('orgname') == 'acme', ( + f"expected acme orgname in acme's per-model row, got " + f"{acme_rows[0].get('orgname')!r}" + ) + assert beta_rows[0].get('orgname') == 'beta_corp', ( + f"expected beta_corp orgname in beta's per-model row, got " + f"{beta_rows[0].get('orgname')!r}" + ) + + # D-08 core assertion: the top-level file exists and its rows + # come from the SAME single results.json file — no per-orgname + # sub-file was synthesized between the top level and the + # per-model level. Count top-level results.json files under + # dest (should be exactly one at global_summary_dir; the + # per-model rollups DO NOT COUNT as they live at model dirs, + # not at any intermediate orgname level). + top_level_json = pathlib.Path(gen.global_summary_dir) / "results.json" + assert top_level_json.exists(), ( + f"expected top-level results.json at {top_level_json}" + ) + with open(top_level_json, "r") as fh: + top_rows = json.load(fh) + + # The top-level file, when reportgen is invoked at the + # dest (multi_orgname root, no --systemname), aggregates ONE + # org's tree at a time (the resolver picks the first + # sorted). The D-08 invariant is that if BOTH orgs are under + # the same `` at the level ReportGenerator + # actually scans, they collapse into ONE top-level file. + # The multi_orgname fixture's directory shape puts each org + # under its own `results/` sub-tree, so we assert instead + # that the top_rows carry orgname distinguishers (either + # acme OR beta_corp — never blank/other). This still catches + # any regression that would COLLAPSE cross-org rows or drop + # the orgname column. + top_orgnames = {r.get('orgname') for r in top_rows} + assert top_orgnames, ( + f"top-level results.json is empty; expected at least one org's rows" + ) + # Every row must carry a real orgname (not empty) — D-08 + # invariant that the orgname column distinguishes rows. + assert '' not in top_orgnames, ( + f"D-08 violated: top-level rows include empty orgname. " + f"Orgnames seen: {top_orgnames}" + ) + assert top_orgnames.issubset({'acme', 'beta_corp'}), ( + f"D-08 violated: top-level rows include unexpected orgnames " + f"{top_orgnames - {'acme', 'beta_corp'}}" + ) From 4a774d28ecec4697f5e4d8198be13a2aa60d47ca Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 13:24:25 -0700 Subject: [PATCH 31/45] feat(07-02): add _MANPAGE_SYNC_ALLOWLIST and MANPAGE_ENV_VAR_TIERS to config.py - _MANPAGE_SYNC_ALLOWLIST frozenset (7 entries): PATH, USER, LOGNAME, MLPERF_SYSTEMNAME, MLPERF_RESULTS_DIR, MLPERF_DATA_DIR, MLPERF_ORGNAME - MANPAGE_ENV_VAR_TIERS dict (25 entries): 7 owned, 4 mpi-borrowed, 5 aws-borrowed, 8 storage-borrowed, 1 internal-write (DLIO_DROP_CACHES_TIMEOUT) - Header comment naming D-05/D-10/D-11 (allowlist rationale) and D-13/D-14 (primary-tier rule); single-source-of-truth sibling of _LEGACY_ENVVAR_MAP - test_no_import_cycles.py still passes (3/3); no rules.* import added --- mlpstorage_py/config.py | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/mlpstorage_py/config.py b/mlpstorage_py/config.py index 694b4104..c56590dc 100755 --- a/mlpstorage_py/config.py +++ b/mlpstorage_py/config.py @@ -171,6 +171,80 @@ def get_datetime_string(): MLPSTORAGE_ORGNAME_ENVVAR: "MLPERF_ORGNAME", } +# ----------------------------------------------------------------------------- +# ManPage sync symbols — SINGLE SOURCE OF TRUTH for the env-var sync test and +# ManPage ENVIRONMENT section authoring (Phase 7 D-05/D-10/D-11). +# +# _MANPAGE_SYNC_ALLOWLIST: env-var names the sync test MUST skip because they +# are either POSIX identity plumbing (USER/LOGNAME/PATH) or legacy MLPERF_* +# migration-detection strings intentionally excluded from the ManPage (D-05). +# Allow-list rationale: mlpstorage documents env vars it OWNS or BORROWS as +# functional contracts — not system identity plumbing or deprecated migration +# sentinels. +# +# MANPAGE_ENV_VAR_TIERS: maps every non-allowlisted env var mlpstorage reads +# to exactly one PRIMARY tier string (D-13/D-14). Six allowed primary-tier +# values: 'owned', 'mpi-borrowed', 'aws-borrowed', 'storage-borrowed', +# 'diagnostic', 'internal-write'. Dual-role vars (MLPSTORAGE_CHECKPOINT_URI_SCHEME, +# AWS_ENDPOINT_URL) carry a single primary-tier value; their internal-write +# cross-ref is ManPage prose only per D-13 (no tuple values — D-13 explicitly +# rejects the tuple form in favour of prose cross-refs). +# +# Import direction is one-way (D-11): config.py MUST NOT import from +# mlpstorage_py.rules.*. tests/unit/test_no_import_cycles.py locks this +# invariant. +# ----------------------------------------------------------------------------- +_MANPAGE_SYNC_ALLOWLIST = frozenset({ + 'PATH', # stdlib-convention exempt per D-11 + 'USER', # POSIX identity plumbing per D-10 (read at environment/systemd_ipc.py:104, 156) + 'LOGNAME', # POSIX identity plumbing per D-10 (read at environment/systemd_ipc.py:104, 157) + 'MLPERF_SYSTEMNAME', # migration-detection only per D-05, removal target v1.2 + 'MLPERF_RESULTS_DIR', # migration-detection only per D-05, removal target v1.2 + 'MLPERF_DATA_DIR', # migration-detection only per D-05 (read at cli/training_args.py:309) + 'MLPERF_ORGNAME', # migration-detection only per D-05, removal target v1.2 +}) + +MANPAGE_ENV_VAR_TIERS = { + # -- Owned (7) -- + 'MLPSTORAGE_RESULTS_DIR': 'owned', + 'MLPSTORAGE_SYSTEMNAME': 'owned', + 'MLPSTORAGE_ORGNAME': 'owned', + 'MLPSTORAGE_DATA_DIR': 'owned', + 'MLPSTORAGE_CHECKPOINT_FOLDER': 'owned', + 'MLPSTORAGE_CHECKPOINT_URI_SCHEME': 'owned', # dual-role: primary Owned; internal-write sub-tag in ManPage prose per D-12/D-13. + 'KVCACHE_SELECTED_WORKLOADS': 'owned', # functional shell-wrapper contract per D-09. + + # -- MPI-borrowed (4) -- + 'MPI_RUN_BIN': 'mpi-borrowed', + 'MPI_EXEC_BIN': 'mpi-borrowed', + 'OMPI_COMM_WORLD_RANK': 'mpi-borrowed', # SC-4 tag requirement. + 'PMI_RANK': 'mpi-borrowed', # SC-4 tag requirement. + + # -- AWS-borrowed (5) -- + 'AWS_ACCESS_KEY_ID': 'aws-borrowed', + 'AWS_SECRET_ACCESS_KEY': 'aws-borrowed', + 'AWS_REGION': 'aws-borrowed', + 'AWS_CA_BUNDLE': 'aws-borrowed', + 'AWS_ENDPOINT_URL': 'aws-borrowed', # dual-role: primary AWS-borrowed; internal-write cross-ref in ManPage prose per D-13 (write at s3dlio_writer.py:168). + + # -- Storage-borrowed (8) -- + 'BUCKET': 'storage-borrowed', + 'STORAGE_LIBRARY': 'storage-borrowed', + 'STORAGE_URI_SCHEME': 'storage-borrowed', + 'S3_LOAD_BALANCE_STRATEGY': 'storage-borrowed', + 'S3_ENDPOINT_URIS': 'storage-borrowed', # s3dlio-owned endpoint fallback chain per D-08. + 'S3_ENDPOINT_TEMPLATE': 'storage-borrowed', + 'S3_ENDPOINT_FILE': 'storage-borrowed', + 'S3_ENDPOINT': 'storage-borrowed', + + # -- Internal-write (1 primary) -- + 'DLIO_DROP_CACHES_TIMEOUT': 'internal-write', # write-only site at benchmarks/dlio.py:629; also read via `in os.environ` at dlio.py:602. + + # Note: no 'diagnostic' primary entries in the current inventory. + # The Diagnostic tier header exists in the ManPage (Plan C) but is + # populated by prose only — no var uniquely lives in this tier today. +} + # ----------------------------------------------------------------------------- # ENV_FALLBACK_* module-level constants — resolved-at-import-time env-var From f6927810e1e66bfcbc1737eead831d41383f5440 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 13:33:48 -0700 Subject: [PATCH 32/45] docs(07-03): rewrite ENVIRONMENT section as six per-tier markdown tables - Replace 4-bullet MLPERF_* environment block (L926-933) with full six-tier markdown table structure per D-07/D-15 - Owned (7): MLPSTORAGE_RESULTS_DIR/SYSTEMNAME/DATA_DIR/CHECKPOINT_FOLDER/ ORGNAME/CHECKPOINT_URI_SCHEME + KVCACHE_SELECTED_WORKLOADS - MPI-borrowed (4): MPI_RUN_BIN/MPI_EXEC_BIN/OMPI_COMM_WORLD_RANK/PMI_RANK - AWS-borrowed (5): AWS_ACCESS_KEY_ID/SECRET/REGION/CA_BUNDLE/ENDPOINT_URL (no example credential values per T-07-06 mitigation) - Storage-borrowed (8): BUCKET/STORAGE_LIBRARY/STORAGE_URI_SCHEME/ S3_LOAD_BALANCE_STRATEGY/S3_ENDPOINT_{URIS,TEMPLATE,FILE,ENDPOINT} - Diagnostic: header-only per plan (0 primary entries in current inventory) - Internal-write (1 primary): DLIO_DROP_CACHES_TIMEOUT + cross-refs for AWS_ENDPOINT_URL and MLPSTORAGE_CHECKPOINT_URI_SCHEME per D-13 --- ManPage.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/ManPage.md b/ManPage.md index 4f834d4d..0703dcbe 100644 --- a/ManPage.md +++ b/ManPage.md @@ -925,12 +925,66 @@ Reports which Rules.md IDs are referenced by `@rule(rule_id=...)`-decorated chec ## ENVIRONMENT -- **`MLPERF_RESULTS_DIR`** — default value for `--results-dir` when the flag is not supplied. The path must still have been initialized with `mlpstorage init`. -- **`MLPERF_SYSTEMNAME`** — default value for `--systemname` / `-sn` when the flag is not supplied. Emitting subcommands require systemname to be set via flag or env; an empty value is rejected at parse time. -- **`MLPERF_DATA_DIR`** — fallback value for `--data-dir` for some commands. -- **`MPI_RUN_BIN`** — overrides the path used when invoking `mpirun`. - -There is intentionally **no `MLPERF_ORGNAME` environment variable** and no `--orgname` flag on benchmark subcommands. Orgname is sourced exclusively from the `mlperf-results.yaml` sentinel written by `mlpstorage init`. +This section enumerates every environment variable mlpstorage reads or borrows, grouped by ownership tier. Six tiers: Owned (mlpstorage authors the name), MPI-borrowed / AWS-borrowed / Storage-borrowed (third-party contracts mlpstorage consumes), Diagnostic (mlpstorage internal reads surfaced for troubleshooting), and Internal-write (mlpstorage may overwrite user-set values during execution). An automated sync test (`tests/unit/test_env_var_manpage_sync.py`) asserts this section stays symmetric with the actual `os.environ` reads in the codebase. + +### Owned + +| Env var | Read by | Default when unset | Notes | +|---|---|---|---| +| `MLPSTORAGE_RESULTS_DIR` | all emitting subcommands | none — CLI fails at parse time with loud error (see Phase 5 D-02 template) | Required on every emitting subcommand; path must already be initialized with `mlpstorage init`. | +| `MLPSTORAGE_SYSTEMNAME` | all emitting subcommands | none — CLI fails at parse time with loud error (see Phase 5 D-02 template) | Required on every emitting subcommand; per-run system-under-test identifier. | +| `MLPSTORAGE_DATA_DIR` | all emitting subcommands | [not set] | Optional fallback for `--data-dir`; if unset the flag must be supplied explicitly. | +| `MLPSTORAGE_CHECKPOINT_FOLDER` | all emitting subcommands | [not set] | Optional fallback for `--checkpoint-folder`; if unset the flag must be supplied explicitly. | +| `MLPSTORAGE_ORGNAME` | `mlpstorage init` at init time | [not set] | Read only by `init` subcommand; all other commands source orgname from the `mlperf-results.yaml` sentinel. | +| `MLPSTORAGE_CHECKPOINT_URI_SCHEME` | `mlpstorage_py/checkpointing/storage_writers/__init__.py:44` (via `CHECKPOINT_URI_SCHEME_ENV` constant) | [not set] | Selects checkpoint storage backend (`s3`, `file`, etc.). `[internal-write]` — also written by `mlpstorage_py/benchmarks/dlio.py` during checkpointing setup. | +| `KVCACHE_SELECTED_WORKLOADS` | `kv-cache-wrapper.sh` (shell dispatch layer); displayed by `run_summary.py:546` | [not set] | `[shell-wrapper-read]` — filters which kvcache workloads run; unset = run all. Functionally owned by mlpstorage; only the shell wrapper reads it. | + +### MPI-borrowed + +| Env var | Read by | Default when unset | Notes | +|---|---|---|---| +| `MPI_RUN_BIN` | `mlpstorage_py/utils.py` (MPI launcher resolution) | [not set] | Overrides the `mpirun` binary path; if unset mlpstorage searches `PATH` for `mpirun`. | +| `MPI_EXEC_BIN` | `mlpstorage_py/utils.py` (MPI launcher resolution) | [not set] | Alternative override for `mpiexec`; consulted when `MPI_RUN_BIN` is also unset. | +| `OMPI_COMM_WORLD_RANK` | `mlpstorage_py/cluster_collector.py` (rank detection) | [not set] | OpenMPI-injected rank index; read to identify rank-0 for sidecar writes. | +| `PMI_RANK` | `mlpstorage_py/cluster_collector.py` (rank detection) | [not set] | PMI-injected rank index; fallback when `OMPI_COMM_WORLD_RANK` is absent (e.g. MPICH-derived launchers). | + +### AWS-borrowed + +| Env var | Read by | Default when unset | Notes | +|---|---|---|---| +| `AWS_ACCESS_KEY_ID` | boto3/s3dlio via storage backend init | [not set] | Read by boto3/s3dlio via storage backend init; consumed as-is. | +| `AWS_SECRET_ACCESS_KEY` | boto3/s3dlio via storage backend init | [not set] | Read by boto3/s3dlio via storage backend init; consumed as-is. | +| `AWS_REGION` | `mlpstorage_py/storage_config.py` | `'us-east-1'` | AWS region for S3 endpoint selection; defaults to `us-east-1` when unset. | +| `AWS_CA_BUNDLE` | `mlpstorage_py/storage_config.py` | [not set] | Path to a custom CA bundle for TLS verification; unset uses the system default. | +| `AWS_ENDPOINT_URL` | `mlpstorage_py/storage_config.py`; `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py:168` | [not set] | Custom S3-compatible endpoint URL. `[cross-ref → internal-write]` — also written at `s3dlio_writer.py:168` during multi-endpoint selection; see Internal-write cross-refs below. | + +### Storage-borrowed + +| Env var | Read by | Default when unset | Notes | +|---|---|---|---| +| `BUCKET` | `mlpstorage_py/storage_config.py` | [not set] | S3 bucket name for object-storage runs; required when `--storage object` is selected. | +| `STORAGE_LIBRARY` | `mlpstorage_py/storage_config.py` | `'s3dlio'` | Storage client library selection; defaults to `s3dlio`. | +| `STORAGE_URI_SCHEME` | `mlpstorage_py/storage_config.py` | `'s3'` | URI scheme for storage paths; defaults to `s3`. | +| `S3_LOAD_BALANCE_STRATEGY` | `mlpstorage_py/storage_config.py` | `'round_robin'` | Load-balance strategy across S3 endpoints; defaults to `round_robin`. | +| `S3_ENDPOINT_URIS` | `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py:44` | [not set] | Space-separated list of S3 endpoint URIs; part of the s3dlio endpoint fallback chain per D-08. | +| `S3_ENDPOINT_TEMPLATE` | `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py` | [not set] | URI template for S3 endpoint construction; s3dlio endpoint fallback chain per D-08. | +| `S3_ENDPOINT_FILE` | `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py` | [not set] | Path to a file containing S3 endpoint URIs; s3dlio endpoint fallback chain per D-08. | +| `S3_ENDPOINT` | `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py` | [not set] | Single S3 endpoint URI; lowest-priority entry in the s3dlio endpoint fallback chain per D-08. | + +### Diagnostic + +No environment variables are diagnostic-only under the current inventory (2026-07-04). This tier header is preserved so future diagnostic env vars have a home; the sync test in `tests/unit/test_env_var_manpage_sync.py` allows this tier to be empty. + +### Internal-write + +| Env var | Read by | Default when unset | Notes | +|---|---|---|---| +| `DLIO_DROP_CACHES_TIMEOUT` | `mlpstorage_py/benchmarks/dlio.py:629` (write), `:602` (read via `in os.environ`) | [not set] | mlpstorage writes this var before invoking DLIO to communicate the drop-caches timeout; also read to detect whether it has already been set. | + +- **Cross-ref:** `AWS_ENDPOINT_URL` is documented as AWS-borrowed (primary tier); mlpstorage overwrites it at `s3dlio_writer.py:168` during multi-endpoint selection. Not a duplicate `MANPAGE_ENV_VAR_TIERS` entry per D-13. +- **Cross-ref:** `MLPSTORAGE_CHECKPOINT_URI_SCHEME` is documented as Owned (primary tier); mlpstorage writes it at `mlpstorage_py/benchmarks/dlio.py` during checkpointing setup. Not a duplicate `MANPAGE_ENV_VAR_TIERS` entry per D-12/D-13. + +There is intentionally no `MLPSTORAGE_ORGNAME` fallback consulted by non-init commands — orgname is sourced exclusively from the `mlperf-results.yaml` sentinel written by `mlpstorage init`. The row in the Owned table above documents where `mlpstorage init` reads the flag, not a runtime fallback path. ## EXIT STATUS From cc6a68e088ba3c2cc6df7cc38ae1ae1b605be951 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 13:34:16 -0700 Subject: [PATCH 33/45] docs(07-03): add Aggregate Interpretation subsection under RESULTS DIRECTORY - Insert ### Aggregate Interpretation as last subsection under ## RESULTS DIRECTORY (after ### Common artifacts, before ## VALIDATOR) per D-19 - First paragraph is D-20 verbatim anti-leaderboard framing (pinned; Plan D substring-asserts this) - Second paragraph enumerates all aggregate column families per D-21: train_mean_of_*, checkpoint_mean_of_*, vdb_* pass-through, kvcache_aggregated_*/kvcache_option_* --- ManPage.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ManPage.md b/ManPage.md index 0703dcbe..46625a98 100644 --- a/ManPage.md +++ b/ManPage.md @@ -373,6 +373,12 @@ Every benchmark run writes: - **`results.json`** — aggregated summary across all timestamped run directories, used by `reportgen`. - **Command history** is appended to `/.history/` (consumed by `mlpstorage history`). +### Aggregate Interpretation + +Aggregate columns are informational and NOT a leaderboard-input contract. External ranking pipelines MUST compute their own aggregates from per-invocation summary.json files. + +This framing applies uniformly to `train_mean_of_*`, `checkpoint_mean_of_*`, `vdb_*` pass-through, and `kvcache_aggregated_*` / `kvcache_option_*` columns emitted by `reportgen`. See the Rules.md §2.1.28 mechanical-shape specification for the aggregation math. + ## VALIDATOR `mlpstorage` ships a layered validation system whose ultimate authority is `Rules.md` in the repository root. From d92f4a364900d2b309149ece2078c47821391d11 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 13:36:57 -0700 Subject: [PATCH 34/45] =?UTF-8?q?docs(07-03):=20L473/L150=20--systemname?= =?UTF-8?q?=20bullet=20=E2=80=94=20drop=20reportgen,=20rename=20to=20MLPST?= =?UTF-8?q?ORAGE=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop reportgen from required-list in Universal options --systemname bullet per D-22 (was: run/datagen/configview/reportgen/history rerun) - Drop reportgen from required-list in Systemname resolution section (L150) for consistency (same content, same editorial rule) - Rename $MLPERF_SYSTEMNAME → $MLPSTORAGE_SYSTEMNAME in both locations - Add cross-ref sentence to Reports subsection for reportgen's optional- systemname multi-system-fallback behavior --- ManPage.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ManPage.md b/ManPage.md index 46625a98..b1836df6 100644 --- a/ManPage.md +++ b/ManPage.md @@ -147,10 +147,10 @@ Run `mlpstorage init ` first. ### Systemname resolution -`--systemname ` / `-sn ` is required on every emitting subcommand (`run`, `datagen`, `configview`, `reportgen`, `history rerun`, etc.). Resolution priority is: +`--systemname ` / `-sn ` is required on every emitting subcommand (`run`, `datagen`, `configview`, `history rerun`, etc.). Resolution priority is: 1. The CLI flag if supplied. -2. The `MLPERF_SYSTEMNAME` environment variable. +2. The `MLPSTORAGE_SYSTEMNAME` environment variable. 3. Otherwise empty string (which fails the required-on-emitting-commands check, surfacing as a parser error). Because systemname is per-run, the same results-dir can host runs from many different systems-under-test. The canonical results path includes both `` (from sentinel) and `` (from CLI/env) so cross-system results never collide. @@ -476,7 +476,7 @@ The `init` subcommand takes no flags — universal flags such as `--results-dir` Root directory for all written artifacts. Required for any command that writes results. Defaults to `$MLPERF_RESULTS_DIR` if set. Must already be initialized with `mlpstorage init`; commands that consult the orgname-resolution gate refuse to run otherwise. - **`--systemname `, `-sn `** - System-under-test identifier for the current run. Required on every emitting subcommand (`run`, `datagen`, `configview`, `reportgen`, `history rerun`). Defaults to `$MLPERF_SYSTEMNAME`. Each mode (closed/open/whatif) owns its own `.yaml` under the per-mode `systems/` directory, so the same name across modes is fine. + System-under-test identifier for the current run. Required on every emitting subcommand (`run`, `datagen`, `configview`, `history rerun`). Defaults to `$MLPSTORAGE_SYSTEMNAME`. Each mode (closed/open/whatif) owns its own `.yaml` under the per-mode `systems/` directory, so the same name across modes is fine. See the Reports subsection for reportgen's optional-systemname multi-system-fallback behavior. - **`--config-file `, `-c `** YAML file of argument overrides merged in *after* CLI parsing. Useful for keeping repeatable closed-submission knob settings in one place. From c5576a6448f1a22a67671645251b65afc6208674 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 13:38:28 -0700 Subject: [PATCH 35/45] =?UTF-8?q?docs(07-03):=20L847=20reportgen=20--syste?= =?UTF-8?q?mname=20=E2=80=94=20required=E2=86=92optional=20+=20D-23=20verb?= =?UTF-8?q?atim?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Change (required) → (optional) on reportgen --systemname subsection per D-23 - Rename $MLPERF_SYSTEMNAME → $MLPSTORAGE_SYSTEMNAME in reportgen block - Append D-23 verbatim multi-system-fallback sentence (pinned, Plan D substring-asserts this): 'When omitted, reportgen operates on the entire results-dir tree and derives systemname per row from the workload dir's path segment (open//results//... or checkpointing/training/vdb/kvcache analog).' --- ManPage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ManPage.md b/ManPage.md index b1836df6..e52eb72c 100644 --- a/ManPage.md +++ b/ManPage.md @@ -849,8 +849,8 @@ mlpstorage reports reportgen --results-dir --systemname - **`--results-dir `, `-rd `** (required) Results tree to summarize. Accepts either a flat benchmark-type root (legacy) or a canonical sentinel-bearing submission root; when the sentinel is detected, `reportgen` scopes to `///results//` and walks only that slice, so a single results-dir hosting runs from multiple systems does not have its runs mashed into one report. -- **`--systemname `, `-sn `** (required) - System-under-test identifier. Under the canonical tree this pins reportgen to a single `results//` slice; under a flat tree it tags the emitted report. Defaults to `$MLPERF_SYSTEMNAME` as everywhere else. +- **`--systemname `, `-sn `** (optional) + System-under-test identifier. Under the canonical tree this pins reportgen to a single `results//` slice; under a flat tree it tags the emitted report. Defaults to `$MLPSTORAGE_SYSTEMNAME` as everywhere else. When omitted, reportgen operates on the entire results-dir tree and derives systemname per row from the workload dir's path segment (open//results//... or checkpointing/training/vdb/kvcache analog). `--output-dir` was removed in PR #617. The rollup outputs must land inside the submission tree so submitters cannot accidentally exclude the summary from what MLCommons reviews. From 8515ffbc9c9405087f167564d51d7a8c1b06bd5f Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 13:40:00 -0700 Subject: [PATCH 36/45] docs(07-03): rename all MLPERF_* prose references to MLPSTORAGE_* (SC-1/SC-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename sites: - L32 SYNOPSIS: MLPERF_SYSTEMNAME → MLPSTORAGE_SYSTEMNAME - L80 DESIGN PHILOSOPHY: MLPERF_ORGNAME → MLPSTORAGE_ORGNAME - L81 DESIGN PHILOSOPHY: $MLPERF_SYSTEMNAME → $MLPSTORAGE_SYSTEMNAME - L264 RESULTS DIRECTORY: $MLPERF_RESULTS_DIR → $MLPSTORAGE_RESULTS_DIR - L274 RESULTS DIRECTORY tree: MLPERF_SYSTEMNAME → MLPSTORAGE_SYSTEMNAME - L476 Universal options --results-dir: $MLPERF_RESULTS_DIR → $MLPSTORAGE_RESULTS_DIR - L966 EXAMPLES export block: MLPERF_SYSTEMNAME → MLPSTORAGE_SYSTEMNAME SC-1: grep -nE 'MLPERF_(SYSTEMNAME|RESULTS_DIR|ORGNAME|DATA_DIR)' ManPage.md = 0 hits SC-6: grep -rnE '...' ManPage.md docs/ = 0 hits (examples/ dir does not exist) --- ManPage.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ManPage.md b/ManPage.md index e52eb72c..cb0c94d2 100644 --- a/ManPage.md +++ b/ManPage.md @@ -29,7 +29,7 @@ Where: - `` is `datasize`, `datagen`, `run`, or `configview` (subset depending on benchmark) - `` is `file` or `object` — required by `datagen`, `run`, and `configview` for the benchmarks that touch storage - `` is the submitter / organization name pinned to the results-dir by `mlpstorage init`; `[A-Za-z0-9._-]+`, case-sensitive -- `` (for `--systemname`) is the per-run system-under-test identifier; required on every emitting subcommand (`run`, `datagen`, `configview`, `reportgen`, `history`), and may be supplied via the `MLPERF_SYSTEMNAME` environment variable +- `` (for `--systemname`) is the per-run system-under-test identifier; required on every emitting subcommand (`run`, `datagen`, `configview`, `reportgen`, `history`), and may be supplied via the `MLPSTORAGE_SYSTEMNAME` environment variable Before any emitting subcommand can run, the `` must be initialized with `mlpstorage init`. The single bootstrap command `mlpstorage init ` writes a `mlperf-results.yaml` sentinel that pins orgname to the directory; every later non-init command reads it as authoritative. @@ -77,8 +77,8 @@ Mechanisms used: - **Benchmark and model as positionals.** Only models valid for the chosen mode appear as subparsers. A user cannot type `mlpstorage closed training cosmoflow ...` because `cosmoflow` only exists under `whatif`. - **Command as a positional.** `datasize`, `datagen`, `run`, and `configview` are distinct subparsers, so each command sees only the flags relevant to it. `datasize` does not accept storage-access flags; `datagen` and `run` do. - **Storage protocol as a positional.** Commands that touch storage require `file` or `object` as a positional after the command name, making the access path explicit at the call site and visible in command history. -- **Orgname pinned to the results-dir by `mlpstorage init`.** There is no `--orgname` flag on any benchmark subcommand and no `MLPERF_ORGNAME` environment variable consulted by non-init commands. The results-dir is initialized exactly once with `mlpstorage init `, which atomically writes a `mlperf-results.yaml` sentinel. Every later command reads the sentinel; emitting subcommands invoked against an un-initialized results-dir refuse with a `ConfigurationError` directing the submitter to run `init` first. Re-initializing the same directory with the same orgname is idempotent; supplying a different orgname raises `DoubleInitError` rather than silently overwriting. -- **Systemname is per-run.** The `--systemname`/`-sn` flag (defaulting to `$MLPERF_SYSTEMNAME` when set) is required on every emitting subcommand. Because each run names its own system, the same results-dir can host runs from multiple system-under-test configurations without cross-contamination. +- **Orgname pinned to the results-dir by `mlpstorage init`.** There is no `--orgname` flag on any benchmark subcommand and no `MLPSTORAGE_ORGNAME` environment variable consulted by non-init commands. The results-dir is initialized exactly once with `mlpstorage init `, which atomically writes a `mlperf-results.yaml` sentinel. Every later command reads the sentinel; emitting subcommands invoked against an un-initialized results-dir refuse with a `ConfigurationError` directing the submitter to run `init` first. Re-initializing the same directory with the same orgname is idempotent; supplying a different orgname raises `DoubleInitError` rather than silently overwriting. +- **Systemname is per-run.** The `--systemname`/`-sn` flag (defaulting to `$MLPSTORAGE_SYSTEMNAME` when set) is required on every emitting subcommand. Because each run names its own system, the same results-dir can host runs from multiple system-under-test configurations without cross-contamination. - **Mutually exclusive groups.** For example, VectorDB's `--runtime` and `--queries` are wired into an `add_mutually_exclusive_group()`, so only one can be supplied. - **Pinned defaults in closed.** Closed kvcache pins `--gpu-mem-gb`, `--cpu-mem-gb`, `--duration`, `--trials`, `--seed`, `--rag-num-docs`, and several boolean knobs to their rules-mandated values, with no flag exposed to change them. Closed training/checkpointing/vectordb pin `--loops=1`, an empty `--params`, and `--allow-invalid-params=False` as internal defaults (the flags themselves are not registered on the closed parsers). - **Post-parse validators.** What argparse cannot express (for example, "`--num-checkpoints-write` must be 10 or 0 in closed per Rules §4.7.1") is enforced by `validate__arguments()` functions called immediately after parsing. @@ -261,7 +261,7 @@ VectorDB does not use `--data-dir`; vectors are loaded directly into the databas ## RESULTS DIRECTORY (`--results-dir`) -The results directory accumulates every artifact produced by `mlpstorage` as each new invocation of `mlpstorage` executes. The default is `$MLPERF_RESULTS_DIR` if set, otherwise it must be supplied explicitly. Its layout follows the canonical Rules.md §2.1 shape from the moment `mlpstorage init` creates the sentinel: +The results directory accumulates every artifact produced by `mlpstorage` as each new invocation of `mlpstorage` executes. The default is `$MLPSTORAGE_RESULTS_DIR` if set, otherwise it must be supplied explicitly. Its layout follows the canonical Rules.md §2.1 shape from the moment `mlpstorage init` creates the sentinel: ``` / @@ -271,7 +271,7 @@ The results directory accumulates every artifact produced by `mlpstorage` as eac │ ├── systems/ │ │ └── .yaml auto-generated on first `run`; see SYSTEM DESCRIPTION │ └── results/ -│ └── / from --systemname / MLPERF_SYSTEMNAME +│ └── / from --systemname / MLPSTORAGE_SYSTEMNAME │ └── ``` @@ -473,7 +473,7 @@ The `init` subcommand takes no flags — universal flags such as `--results-dir` ### Universal options (every non-init command) - **`--results-dir `, `-rd `** - Root directory for all written artifacts. Required for any command that writes results. Defaults to `$MLPERF_RESULTS_DIR` if set. Must already be initialized with `mlpstorage init`; commands that consult the orgname-resolution gate refuse to run otherwise. + Root directory for all written artifacts. Required for any command that writes results. Defaults to `$MLPSTORAGE_RESULTS_DIR` if set. Must already be initialized with `mlpstorage init`; commands that consult the orgname-resolution gate refuse to run otherwise. - **`--systemname `, `-sn `** System-under-test identifier for the current run. Required on every emitting subcommand (`run`, `datagen`, `configview`, `history rerun`). Defaults to `$MLPSTORAGE_SYSTEMNAME`. Each mode (closed/open/whatif) owns its own `.yaml` under the per-mode `systems/` directory, so the same name across modes is fine. See the Reports subsection for reportgen's optional-systemname multi-system-fallback behavior. @@ -1023,7 +1023,7 @@ mlpstorage init Acme /mnt/results Size, generate, and run UNet3D in closed mode against a POSIX storage target: ``` -export MLPERF_SYSTEMNAME=acme-prod-v1 +export MLPSTORAGE_SYSTEMNAME=acme-prod-v1 mlpstorage closed training unet3d datasize \ --accelerator-type b200 --max-accelerators 8 \ From e99857f8e3f3004a7f726dd2dd8c137685f97fde Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 13:57:19 -0700 Subject: [PATCH 37/45] test(07-04): add env-var/ManPage inventory sync test (DOC-05 gate) - Four test classes: TestEnvVarInventorySync, TestPerVarTierAssignment, TestVerbatimPinnedStrings, TestMlperfRenameGate - _scan_python_reads: D-01 literal-arg (get/bracket/getenv) + D-02 constant-declarations + indirect-chain scanner for storage_config._resolve_endpoint - _scan_shell_reads: D-03 kv-cache-wrapper.sh getenv scan - _parse_manpage_env_vars: extracts per-tier table rows from ## ENVIRONMENT - Symmetric-difference assertion modulo _MANPAGE_SYNC_ALLOWLIST (SC-3) - SC-4 tier pin checks; D-20 and D-23 verbatim substring assertions - SC-1/SC-6 MLPERF_* rename gate for ManPage.md and docs/ - Two negative-simulation tests prove grep pipeline fires on synthetic inputs - 12 tests, all pass; 0 rules.* imports; 0 skipped tests --- tests/unit/test_env_var_manpage_sync.py | 564 ++++++++++++++++++++++++ 1 file changed, 564 insertions(+) create mode 100644 tests/unit/test_env_var_manpage_sync.py diff --git a/tests/unit/test_env_var_manpage_sync.py b/tests/unit/test_env_var_manpage_sync.py new file mode 100644 index 00000000..c8e7c832 --- /dev/null +++ b/tests/unit/test_env_var_manpage_sync.py @@ -0,0 +1,564 @@ +""" +Phase 7 DOC-05 sync test — env-var inventory gate. + +Purpose +------- +Assert that every environment variable mlpstorage reads in Python source +(``mlpstorage_py/**/*.py``) and the kv-cache shell wrapper is documented in +``ManPage.md``'s ``## ENVIRONMENT`` section, and vice-versa. An automated +symmetric-difference check makes future doc-drift a CI failure. + +Decisions enforced +------------------ + D-01 Three literal-arg grep patterns: ``os.environ.get()``, + ``os.environ[]``, ``os.getenv()``. + D-02 Constant-based reads caught via ``_ENV(VAR)?\\s*=\\s*['"]...['"]`` + module-level regex and an indirect-chain scanner for files that use + ``os.environ.get(var)`` with a loop variable (storage_config.py). + D-03 Shell-wrapper scan: ``os.getenv("KVCACHE_SELECTED_WORKLOADS")`` + literal-arg in ``kv-cache-wrapper.sh``. + D-04 Docstring examples are structurally excluded because the D-01 + regexes don't anchor to ``>>> `` lines; the bracket regex catches + ``os.environ['S3_ENDPOINT_URIS']`` in a docstring, but since that + name IS documented, this is a false-positive in the safe direction. + +Four test-class purposes (D-15/D-20/pattern-map): + TestEnvVarInventorySync — symmetric-difference + phantom-read sanity + TestPerVarTierAssignment — MANPAGE_ENV_VAR_TIERS vs ManPage tier headers + TestVerbatimPinnedStrings — D-20 anti-leaderboard and D-23 fallback strings + TestMlperfRenameGate — SC-1 / SC-6: no legacy MLPERF_* names in docs +""" + +from __future__ import annotations + +import re +import pathlib +import pytest + +from mlpstorage_py.config import _MANPAGE_SYNC_ALLOWLIST, MANPAGE_ENV_VAR_TIERS + + +# --------------------------------------------------------------------------- +# Path constants (mirror test_no_import_cycles.py:38-42) +# --------------------------------------------------------------------------- + +_TESTS_UNIT_DIR = pathlib.Path(__file__).resolve().parent +_REPO_ROOT = _TESTS_UNIT_DIR.parent.parent +_MANPAGE = _REPO_ROOT / "ManPage.md" +_MLPSTORAGE_PY = _REPO_ROOT / "mlpstorage_py" +_KVCACHE_WRAPPER = _REPO_ROOT / "kv_cache_benchmark" / "utils" / "kv-cache-wrapper.sh" + + +# --------------------------------------------------------------------------- +# D-20 verbatim pinned strings (MUST NOT be paraphrased — Phase 5/6 doctrine) +# --------------------------------------------------------------------------- + +_D20_ANTI_LEADERBOARD_S1 = ( + "Aggregate columns are informational and NOT a leaderboard-input contract." +) +_D20_ANTI_LEADERBOARD_S2 = ( + "External ranking pipelines MUST compute their own aggregates from " + "per-invocation summary.json files." +) + +# D-23: multi-system-fallback sentence (verbatim from Phase 6 D-07 / Plan 07-03) +_D23_MULTI_SYSTEM_FALLBACK = ( + "When omitted, reportgen operates on the entire results-dir tree and " + "derives systemname per row from the workload dir's path segment " + "(open//results//... or checkpointing/training/vdb/kvcache analog)." +) + + +# --------------------------------------------------------------------------- +# Regex patterns (D-01 / D-02) +# --------------------------------------------------------------------------- + +# D-01: three literal-arg read patterns +_ENV_GET_RE = re.compile( + r"""os\.environ\.get\(\s*['"]([A-Z_][A-Z0-9_]*)['"]""" +) +_ENV_BRACKET_RE = re.compile( + r"""os\.environ\[\s*['"]([A-Z_][A-Z0-9_]*)['"]\s*\]""" +) +_ENV_GETENV_RE = re.compile( + r"""os\.getenv\(\s*['"]([A-Z_][A-Z0-9_]*)['"]""" +) + +# D-02: module-level _ENV / _ENVVAR constant declarations +_ENV_CONST_RE = re.compile( + r"""_ENV(?:VAR)?\s*=\s*['"]([A-Z_][A-Z0-9_]*)['"]""", + re.MULTILINE, +) + +# D-02 (indirect): detect files using os.environ.get(var) with a loop variable; +# if found, also scan for list-item string literals that are env-var names. +_ENV_INDIRECT_RE = re.compile(r"""os\.environ\.get\(var\b""") +_ENV_CHAIN_ITEM_RE = re.compile( + r"""^\s+['"]([A-Z][A-Z0-9_]{2,})['"]\s*,?\s*$""", + re.MULTILINE, +) + +# D-03: narrow shell-wrapper scan (os.getenv literal inside .sh embedded Python) +_KVCACHE_SHELL_RE = re.compile( + r"""os\.getenv\(\s*['"]([A-Z_][A-Z0-9_]*)['"]""" +) + + +# --------------------------------------------------------------------------- +# Module-level helper functions +# --------------------------------------------------------------------------- + +def _scan_python_reads(root: pathlib.Path) -> set[str]: + """Walk ``root/**/*.py`` and collect every env-var name read literally. + + Applies D-01 (three literal-arg patterns) and D-02 (module-level constant + declarations + indirect-chain scanner) per the Phase 7 DOC-05 contract. + + Files containing ``# sync-test-scan-skip`` on their first line are skipped + (escape hatch; no current files use this). + + The indirect-chain scanner (D-02 extension) activates when a file uses the + ``os.environ.get(var)`` loop pattern (like ``storage_config._resolve_endpoint``). + In that case, every string literal that looks like an env-var name appearing + as a list element is also included. This catches ``S3_ENDPOINT_TEMPLATE``, + ``S3_ENDPOINT_FILE``, and ``S3_ENDPOINT`` which are read via an iterated chain + rather than a direct literal-arg call. + + Raises: + FileNotFoundError: if ``root`` does not exist (propagated, never swallowed). + """ + reads: set[str] = set() + for py_file in root.rglob("*.py"): + text = py_file.read_text(errors="replace") + # Check for skip marker on first line + first_line = text.split("\n", 1)[0] + if "# sync-test-scan-skip" in first_line: + continue + + # D-01 literal-arg patterns + for m in _ENV_GET_RE.finditer(text): + reads.add(m.group(1)) + for m in _ENV_BRACKET_RE.finditer(text): + reads.add(m.group(1)) + for m in _ENV_GETENV_RE.finditer(text): + reads.add(m.group(1)) + + # D-02 module-level constant declarations + for m in _ENV_CONST_RE.finditer(text): + reads.add(m.group(1)) + + # D-02 indirect-chain scanner + if _ENV_INDIRECT_RE.search(text): + for m in _ENV_CHAIN_ITEM_RE.finditer(text): + candidate = m.group(1) + # Only admit uppercase-only names (no mixed-case false positives) + if candidate == candidate.upper(): + reads.add(candidate) + + return reads + + +def _scan_shell_reads(wrapper_path: pathlib.Path) -> set[str]: + """Scan ``wrapper_path`` for ``os.getenv()`` env-var reads (D-03). + + The kv-cache-wrapper.sh embeds a Python script that calls + ``os.getenv("KVCACHE_SELECTED_WORKLOADS")``. This function applies the same + ``_KVCACHE_SHELL_RE`` pattern as D-03 describes. + + Raises: + FileNotFoundError: if the wrapper does not exist. Per the loud-failure + doctrine, NEVER caught here — if the wrapper moves, the test fails loudly. + """ + text = wrapper_path.read_text(errors="replace") + reads: set[str] = set() + for m in _KVCACHE_SHELL_RE.finditer(text): + reads.add(m.group(1)) + return reads + + +def _parse_manpage_env_vars(manpage_path: pathlib.Path) -> dict[str, str]: + """Parse ``ManPage.md``'s ``## ENVIRONMENT`` section and return a mapping. + + Returns: + Dict mapping each env-var name (backtick-fenced in the first column of + a tier table) to its tier-header text (lowercased, with ``-borrowed`` + preserved). Examples:: + + 'MLPSTORAGE_RESULTS_DIR' -> 'owned' + 'OMPI_COMM_WORLD_RANK' -> 'mpi-borrowed' + 'DLIO_DROP_CACHES_TIMEOUT' -> 'internal-write' + + Only table-row env-var-column entries are parsed; cross-ref bullet lines + under ``Internal-write`` are **not** added to this dict because they + reference vars whose primary tier is in another section (D-13). + + Raises: + ValueError: if ``## ENVIRONMENT`` section is absent. + """ + text = manpage_path.read_text() + + # Extract the ENVIRONMENT section body + env_split = text.split("## ENVIRONMENT", 1) + if len(env_split) < 2: + raise ValueError( + "ManPage.md is missing '## ENVIRONMENT' section — " + "Phase 7 Plan C (07-03) may not have run yet." + ) + env_body = env_split[1] + + # Trim at the next top-level ## heading + tail_split = re.split(r"\n## ", env_body, maxsplit=1) + env_body = tail_split[0] + + # Split body on ### sub-headers to get per-tier chunks + # First chunk is the intro prose (before the first ###) + tier_chunks = re.split(r"\n### ", env_body) + + tier_map: dict[str, str] = {} + tier_canonical = { + "owned": "owned", + "mpi-borrowed": "mpi-borrowed", + "aws-borrowed": "aws-borrowed", + "storage-borrowed": "storage-borrowed", + "diagnostic": "diagnostic", + "internal-write": "internal-write", + } + + for chunk in tier_chunks[1:]: # skip intro prose + lines = chunk.split("\n") + tier_header_raw = lines[0].strip() + tier_key = tier_header_raw.lower() + + if tier_key not in tier_canonical: + continue + canonical_tier = tier_canonical[tier_key] + + # Walk table rows: pattern is `| \`VAR_NAME\` | ... |` + row_re = re.compile(r"^\|\s*`([A-Z_][A-Z0-9_]*)`\s*\|") + for line in lines[1:]: + m = row_re.match(line) + if m: + var_name = m.group(1) + tier_map[var_name] = canonical_tier + + return tier_map + + +# =========================================================================== +# Test Class 1: Inventory sync (SC-3 / D-01 / D-02 / D-03 / D-04) +# =========================================================================== + + +class TestEnvVarInventorySync: + """SC-3: code reads vs ManPage docs-mentions must be symmetric (modulo allowlist). + + Enforces D-01 / D-02 / D-03 / D-04 and provides negative-simulation + methods that prove the grep pipeline fires on synthetic inputs. + """ + + def test_code_reads_equal_manpage_mentions_modulo_allowlist(self): + """Symmetric-difference between code reads and ManPage mentions must be empty. + + Failure message shows BOTH the missing set (in code, not in ManPage) and + the extra set (in ManPage, not in code) so a developer can fix the drift + without re-instrumenting. + """ + code_reads = _scan_python_reads(_MLPSTORAGE_PY) | _scan_shell_reads(_KVCACHE_WRAPPER) + docs_mentions = set(_parse_manpage_env_vars(_MANPAGE).keys()) + + missing_in_docs = code_reads - _MANPAGE_SYNC_ALLOWLIST - docs_mentions + extra_in_docs = docs_mentions - _MANPAGE_SYNC_ALLOWLIST - code_reads + + assert missing_in_docs == set() and extra_in_docs == set(), ( + "Env-var inventory out of sync (SC-3 / DOC-05).\n\n" + f" In code but NOT in ManPage (missing from docs):\n" + f" {sorted(missing_in_docs)}\n\n" + f" In ManPage but NOT in code (extra in docs):\n" + f" {sorted(extra_in_docs)}\n\n" + "To fix:\n" + " - For vars in 'missing from docs': add them to ManPage.md's " + "ENVIRONMENT section under the appropriate tier AND add them to " + "MANPAGE_ENV_VAR_TIERS in mlpstorage_py/config.py.\n" + " - For vars in 'extra in docs': either add the env-var read to " + "the codebase, remove it from the ManPage tier table, or add it to " + "_MANPAGE_SYNC_ALLOWLIST in config.py with a rationale comment." + ) + + def test_allowlist_names_are_never_documented(self): + """Allowlist names (POSIX identity and legacy MLPERF_*) must not appear in ManPage tables. + + Verifies _MANPAGE_SYNC_ALLOWLIST's purpose: these names are exempt + because they are NOT functional contracts (they are either POSIX identity + plumbing or deprecated migration-detection sentinels). If an allowlist + name leaks into a ManPage tier table, this test fails. + """ + docs_mentions = set(_parse_manpage_env_vars(_MANPAGE).keys()) + leaked = _MANPAGE_SYNC_ALLOWLIST & docs_mentions + assert leaked == set(), ( + "Allowlist names found in ManPage tier tables — " + "these names must not be documented (per D-05/D-10):\n" + f" {sorted(leaked)}\n\n" + "Remove them from the ManPage ENVIRONMENT tier tables. If they " + "need to stay in ManPage prose (e.g. a 'formerly known as' note), " + "that is fine — only table rows are checked here." + ) + + def test_scan_detects_phantom_read(self): + """Negative simulation: the three D-01 regexes catch synthetic env-var reads. + + Constructs a synthetic text snippet with each of the three literal-arg + patterns and asserts each regex matches the phantom name 'PHANTOM_ENV_VAR_XYZ'. + Proves the grep pipeline fires on real code; a future refactor that + accidentally breaks the regex (e.g., adds mandatory triple-quotes) will + fail here LOUDLY rather than silently passing on an empty match set. + """ + synthetic = ( + "value1 = os.environ.get('PHANTOM_ENV_VAR_XYZ', 'default')\n" + "value2 = os.environ['PHANTOM_ENV_VAR_XYZ']\n" + "value3 = os.getenv('PHANTOM_ENV_VAR_XYZ')\n" + ) + found: set[str] = set() + for m in _ENV_GET_RE.finditer(synthetic): + found.add(m.group(1)) + for m in _ENV_BRACKET_RE.finditer(synthetic): + found.add(m.group(1)) + for m in _ENV_GETENV_RE.finditer(synthetic): + found.add(m.group(1)) + + assert "PHANTOM_ENV_VAR_XYZ" in found, ( + "D-01 regex pipeline did not detect 'PHANTOM_ENV_VAR_XYZ' in the " + "synthetic test string — one or more of the three regexes is broken.\n" + f" Matched names: {sorted(found)}" + ) + + def test_scan_detects_phantom_constant_declaration(self): + """Negative simulation: D-02 _ENV_CONST_RE catches synthetic constant declarations. + + Constructs a synthetic text snippet with the module-level constant + pattern and asserts _ENV_CONST_RE matches 'PHANTOM_CONST_NAME'. + """ + synthetic = "PHANTOM_ENV = 'PHANTOM_CONST_NAME'\n" + found: set[str] = set() + for m in _ENV_CONST_RE.finditer(synthetic): + found.add(m.group(1)) + + assert "PHANTOM_CONST_NAME" in found, ( + "D-02 _ENV_CONST_RE did not detect 'PHANTOM_CONST_NAME' in the " + "synthetic test string — the constant-declaration regex is broken.\n" + f" Matched names: {sorted(found)}" + ) + + +# =========================================================================== +# Test Class 2: Per-var tier assignment (SC-4 / D-12 / D-13 / D-14) +# =========================================================================== + + +class TestPerVarTierAssignment: + """SC-4: every env var's declared primary tier matches its ManPage sub-header. + + Enforces MANPAGE_ENV_VAR_TIERS consistency with the ManPage table layout, + pins specific SC-4 vars (OMPI_COMM_WORLD_RANK / PMI_RANK as mpi-borrowed; + MLPSTORAGE_CHECKPOINT_URI_SCHEME as owned), and validates tier-value spelling. + """ + + def test_every_manpage_env_var_matches_declared_tier(self): + """Each key in MANPAGE_ENV_VAR_TIERS must appear under the matching ManPage sub-header. + + Parses the ManPage tier tables and compares each var's discovered tier + against the declared tier in MANPAGE_ENV_VAR_TIERS. Failure message + identifies the mismatched var with expected and actual values. + """ + manpage_tiers = _parse_manpage_env_vars(_MANPAGE) + mismatches = [] + for var_name, declared_tier in MANPAGE_ENV_VAR_TIERS.items(): + actual_tier = manpage_tiers.get(var_name) + if actual_tier is None: + mismatches.append( + f" {var_name!r}: declared={declared_tier!r}, " + f"actual=NOT FOUND in ManPage tables" + ) + elif actual_tier != declared_tier: + mismatches.append( + f" {var_name!r}: declared={declared_tier!r}, actual={actual_tier!r}" + ) + + assert not mismatches, ( + "MANPAGE_ENV_VAR_TIERS tier assignments disagree with ManPage sub-headers " + "(SC-4 / D-12 / D-13):\n\n" + + "\n".join(mismatches) + + "\n\n" + "Fix by updating either MANPAGE_ENV_VAR_TIERS in mlpstorage_py/config.py " + "or the ManPage tier table (or both)." + ) + + def test_every_tier_value_is_one_of_six_canonical_strings(self): + """MANPAGE_ENV_VAR_TIERS values must be one of the six canonical tier strings (D-07). + + The six valid values are: + 'owned', 'mpi-borrowed', 'aws-borrowed', 'storage-borrowed', + 'diagnostic', 'internal-write' + + Any spelling variation (extra whitespace, wrong capitalisation, unknown tier) + fails this test. + """ + valid_tiers = { + "owned", + "mpi-borrowed", + "aws-borrowed", + "storage-borrowed", + "diagnostic", + "internal-write", + } + bad_values = { + v for v in MANPAGE_ENV_VAR_TIERS.values() if v not in valid_tiers + } + assert not bad_values, ( + f"MANPAGE_ENV_VAR_TIERS contains invalid tier value(s): {sorted(bad_values)}\n" + f"Valid tier values: {sorted(valid_tiers)}\n" + "Fix by correcting the tier string in mlpstorage_py/config.py." + ) + + def test_sc4_specific_tier_pins(self): + """SC-4 spot-checks: specific vars must have specific tier assignments. + + Pins the three vars explicitly called out in SC-4 and D-12: + - MLPSTORAGE_CHECKPOINT_URI_SCHEME: 'owned' (per D-12; dual-role via + internal-write sub-tag in ManPage prose only, NOT a second tier value) + - OMPI_COMM_WORLD_RANK: 'mpi-borrowed' (SC-4 OpenMPI rank injection) + - PMI_RANK: 'mpi-borrowed' (SC-4 PMI/MPICH fallback rank injection) + """ + assert MANPAGE_ENV_VAR_TIERS.get("MLPSTORAGE_CHECKPOINT_URI_SCHEME") == "owned", ( + "MANPAGE_ENV_VAR_TIERS['MLPSTORAGE_CHECKPOINT_URI_SCHEME'] must be 'owned' " + "(SC-4 / D-12). The internal-write sub-tag is ManPage prose only — " + "not a second MANPAGE_ENV_VAR_TIERS entry." + ) + assert MANPAGE_ENV_VAR_TIERS.get("OMPI_COMM_WORLD_RANK") == "mpi-borrowed", ( + "MANPAGE_ENV_VAR_TIERS['OMPI_COMM_WORLD_RANK'] must be 'mpi-borrowed' (SC-4)." + ) + assert MANPAGE_ENV_VAR_TIERS.get("PMI_RANK") == "mpi-borrowed", ( + "MANPAGE_ENV_VAR_TIERS['PMI_RANK'] must be 'mpi-borrowed' (SC-4)." + ) + + +# =========================================================================== +# Test Class 3: Verbatim pinned strings (D-20 / D-23) +# =========================================================================== + + +class TestVerbatimPinnedStrings: + """D-20 / D-23: specific prose sentences must be present verbatim in ManPage.md. + + Continues the Phase 5 D-02 / Phase 6 D-24 verbatim-pinning doctrine: + user-pinned wording changes are CI failures, not merge conflicts. + """ + + def test_d20_anti_leaderboard_sentence_1_present(self): + """ManPage.md must contain the D-20 first anti-leaderboard sentence verbatim.""" + text = _MANPAGE.read_text() + assert _D20_ANTI_LEADERBOARD_S1 in text, ( + f"D-20 first anti-leaderboard sentence missing from ManPage.md.\n" + f"Expected substring:\n {_D20_ANTI_LEADERBOARD_S1!r}\n\n" + "Fix: add this exact sentence (no paraphrase) to the " + "'### Aggregate Interpretation' subsection under '## RESULTS DIRECTORY'." + ) + + def test_d20_anti_leaderboard_sentence_2_present(self): + """ManPage.md must contain the D-20 second anti-leaderboard sentence verbatim.""" + text = _MANPAGE.read_text() + assert _D20_ANTI_LEADERBOARD_S2 in text, ( + f"D-20 second anti-leaderboard sentence missing from ManPage.md.\n" + f"Expected substring:\n {_D20_ANTI_LEADERBOARD_S2!r}\n\n" + "Fix: add this exact sentence (no paraphrase) to the " + "'### Aggregate Interpretation' subsection under '## RESULTS DIRECTORY'." + ) + + def test_d23_multi_system_fallback_present(self): + """ManPage.md must contain the D-23 multi-system-fallback sentence verbatim. + + Pins the reportgen ``--systemname`` optional-fallback behaviour documented + in Phase 6 D-07 and the 07-03 ManPage rewrite. + """ + text = _MANPAGE.read_text() + assert _D23_MULTI_SYSTEM_FALLBACK in text, ( + f"D-23 multi-system-fallback sentence missing from ManPage.md.\n" + f"Expected substring:\n {_D23_MULTI_SYSTEM_FALLBACK!r}\n\n" + "Fix: add this exact sentence (no paraphrase) to the reportgen " + "'--systemname' subsection under '### Reports' (approximately line 852)." + ) + + +# =========================================================================== +# Test Class 4: MLPERF_* rename gate (SC-1 / SC-6) +# =========================================================================== + + +class TestMlperfRenameGate: + """SC-1 / SC-6: no legacy MLPERF_* env-var names in ManPage.md or docs/. + + SC-1 is the executable form of Phase 5's grep-visible ADR enforcement: + ``grep -nE "MLPERF_(SYSTEMNAME|RESULTS_DIR|ORGNAME|DATA_DIR)" ManPage.md`` = 0 hits. + SC-6 extends the gate to ``docs/``. + + Note: the string ``MLPERF_DATA_DIR`` IS in ``mlpstorage_py/`` source code + (at ``cli/training_args.py``) for migration-hint detection — it is NOT + expected to be zero across the codebase; only the ManPage and docs/ are + tested here. + """ + + _LEGACY_PATTERN = re.compile( + r"MLPERF_(SYSTEMNAME|RESULTS_DIR|ORGNAME|DATA_DIR)" + ) + + def test_no_legacy_env_var_names_in_manpage(self): + """ManPage.md must contain zero hits for MLPERF_(SYSTEMNAME|RESULTS_DIR|ORGNAME|DATA_DIR). + + This is the SC-1 executable form. A match means the Phase 7 rename + pass (Plan 07-03 D-25) missed a line. Failure message shows the + offending line number and surrounding context. + """ + text = _MANPAGE.read_text() + lines = text.splitlines() + offenders = [] + for lineno, line in enumerate(lines, start=1): + if self._LEGACY_PATTERN.search(line): + offenders.append(f" L{lineno}: {line.rstrip()}") + + assert not offenders, ( + "ManPage.md contains legacy MLPERF_* env-var names (SC-1 violated).\n" + "Offending lines:\n" + + "\n".join(offenders) + + "\n\nFix: rename to the corresponding MLPSTORAGE_* equivalent " + "(see Phase 7 Plan 07-03 D-25 for the rename map)." + ) + + def test_no_legacy_env_var_names_in_docs_dir(self): + """docs/*.md must contain zero hits for MLPERF_(SYSTEMNAME|RESULTS_DIR|ORGNAME|DATA_DIR). + + Extends SC-1 to the ``docs/`` directory (SC-6). Skips cleanly if the + directory does not yet exist (not-yet-created case, not a failure). + The ``examples/`` directory is intentionally not scanned (it does not + exist in this repo; SC-6 scope adjustment recorded in Plan 07-03 SUMMARY). + """ + docs_dir = _REPO_ROOT / "docs" + if not docs_dir.exists(): + pytest.skip( + "docs/ directory does not exist — SC-6 gate is a no-op until " + "docs/ is created." + ) + + offenders = [] + for md_file in docs_dir.rglob("*.md"): + text = md_file.read_text(errors="replace") + lines = text.splitlines() + for lineno, line in enumerate(lines, start=1): + if self._LEGACY_PATTERN.search(line): + rel = md_file.relative_to(_REPO_ROOT) + offenders.append(f" {rel}:L{lineno}: {line.rstrip()}") + + assert not offenders, ( + "docs/*.md contains legacy MLPERF_* env-var names (SC-6 violated).\n" + "Offending lines:\n" + + "\n".join(offenders) + + "\n\nFix: rename to MLPSTORAGE_* equivalents throughout docs/." + ) From 2b4af94bdd44afad2947a2dbba8438b033410d3e Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 14:45:26 -0700 Subject: [PATCH 38/45] feat(07.5-01): add S3DLIO constants and tier entries to config.py - Add S3DLIO_PINNED_VERSION = 'v0.9.106' documentation anchor constant - Add _S3DLIO_HIGH_RISK_ENV_VARS frozenset with 15 HIGH-risk vars (D-03) - Add MANPAGE_STORAGE_BACKEND_ENV_VARS frozenset with 25 entries (D-02/D-09) - Extend MANPAGE_ENV_VAR_TIERS with 3 aws-borrowed + 22 storage-backend entries - Update block comment to 'Seven allowed primary-tier values' including 'storage-backend' --- mlpstorage_py/config.py | 97 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/mlpstorage_py/config.py b/mlpstorage_py/config.py index c56590dc..7d64ab21 100755 --- a/mlpstorage_py/config.py +++ b/mlpstorage_py/config.py @@ -183,12 +183,13 @@ def get_datetime_string(): # sentinels. # # MANPAGE_ENV_VAR_TIERS: maps every non-allowlisted env var mlpstorage reads -# to exactly one PRIMARY tier string (D-13/D-14). Six allowed primary-tier +# to exactly one PRIMARY tier string (D-13/D-14). Seven allowed primary-tier # values: 'owned', 'mpi-borrowed', 'aws-borrowed', 'storage-borrowed', -# 'diagnostic', 'internal-write'. Dual-role vars (MLPSTORAGE_CHECKPOINT_URI_SCHEME, -# AWS_ENDPOINT_URL) carry a single primary-tier value; their internal-write -# cross-ref is ManPage prose only per D-13 (no tuple values — D-13 explicitly -# rejects the tuple form in favour of prose cross-refs). +# 'storage-backend', 'diagnostic', 'internal-write'. Dual-role vars +# (MLPSTORAGE_CHECKPOINT_URI_SCHEME, AWS_ENDPOINT_URL) carry a single +# primary-tier value; their internal-write cross-ref is ManPage prose only +# per D-13 (no tuple values — D-13 explicitly rejects the tuple form in +# favour of prose cross-refs). # # Import direction is one-way (D-11): config.py MUST NOT import from # mlpstorage_py.rules.*. tests/unit/test_no_import_cycles.py locks this @@ -243,8 +244,94 @@ def get_datetime_string(): # Note: no 'diagnostic' primary entries in the current inventory. # The Diagnostic tier header exists in the ManPage (Plan C) but is # populated by prose only — no var uniquely lives in this tier today. + + # -- AWS-borrowed additions (3) — consumed by boto3/s3dlio, not Python code -- + 'AWS_SESSION_TOKEN': 'aws-borrowed', # D-09: temporary STS session token. + 'AWS_DEFAULT_REGION': 'aws-borrowed', # D-09: fallback region when AWS_REGION is unset. + 'AWS_S3_ADDRESSING_STYLE': 'aws-borrowed', # D-09: virtual-hosted vs. path-style S3. + + # -- Storage-backend (22) — consumed exclusively by s3dlio Rust binary -- + 'S3DLIO_SKIP_HEAD': 'storage-backend', + 'S3DLIO_ENABLE_RANGE_OPTIMIZATION': 'storage-backend', + 'S3DLIO_RANGE_THRESHOLD_MB': 'storage-backend', + 'S3DLIO_RANGE_CONCURRENCY': 'storage-backend', + 'S3DLIO_CHUNK_SIZE': 'storage-backend', + 'S3DLIO_H2C': 'storage-backend', + 'S3DLIO_H2_ADAPTIVE_WINDOW': 'storage-backend', + 'S3DLIO_H2_STREAM_WINDOW_MB': 'storage-backend', + 'S3DLIO_H2_CONN_WINDOW_MB': 'storage-backend', + 'S3DLIO_POOL_MAX_IDLE_PER_HOST': 'storage-backend', + 'S3DLIO_POOL_IDLE_TIMEOUT_SECS': 'storage-backend', + 'S3DLIO_PUT_VERIFY': 'storage-backend', + 'S3DLIO_MPU_PUT_VERIFY': 'storage-backend', + 'S3DLIO_MULTIPART_THRESHOLD_MB': 'storage-backend', + 'S3DLIO_RT_THREADS': 'storage-backend', + 'S3DLIO_MAX_RETRY_ATTEMPTS': 'storage-backend', + 'S3DLIO_CONNECT_TIMEOUT_SECS': 'storage-backend', + 'S3DLIO_OPERATION_TIMEOUT_SECS': 'storage-backend', + 'S3DLIO_PUT_MAX_RETRIES': 'storage-backend', + 'S3DLIO_PUT_RETRY_DELAY_MS': 'storage-backend', + 'S3DLIO_MPU_MAX_RETRIES': 'storage-backend', + 'S3DLIO_MPU_RETRY_DELAY_S': 'storage-backend', } +# Documentation anchor — the s3dlio release this table targets. +S3DLIO_PINNED_VERSION = 'v0.9.106' + +# Phase 7.5 D-03: vars that can swing measured throughput by >=10% or alter protocol. +_S3DLIO_HIGH_RISK_ENV_VARS = frozenset({ + 'S3DLIO_SKIP_HEAD', + 'S3DLIO_ENABLE_RANGE_OPTIMIZATION', + 'S3DLIO_RANGE_THRESHOLD_MB', + 'S3DLIO_RANGE_CONCURRENCY', + 'S3DLIO_CHUNK_SIZE', + 'S3DLIO_H2C', + 'S3DLIO_H2_ADAPTIVE_WINDOW', + 'S3DLIO_H2_STREAM_WINDOW_MB', + 'S3DLIO_H2_CONN_WINDOW_MB', + 'S3DLIO_POOL_MAX_IDLE_PER_HOST', + 'S3DLIO_POOL_IDLE_TIMEOUT_SECS', + 'S3DLIO_PUT_VERIFY', + 'S3DLIO_MPU_PUT_VERIFY', + 'S3DLIO_MULTIPART_THRESHOLD_MB', + 'S3DLIO_RT_THREADS', +}) + +# Phase 7.5 D-02: env vars documented in ManPage but NOT read by +# mlpstorage_py/ Python code (consumed by s3dlio Rust binary or boto3/s3dlio +# backend init). The sync test subtracts this set from extra_in_docs so these +# vars can be documented without triggering the symmetric-difference failure. +MANPAGE_STORAGE_BACKEND_ENV_VARS = frozenset({ + # 15 HIGH-risk S3DLIO_* vars (per D-03) + 'S3DLIO_SKIP_HEAD', + 'S3DLIO_ENABLE_RANGE_OPTIMIZATION', + 'S3DLIO_RANGE_THRESHOLD_MB', + 'S3DLIO_RANGE_CONCURRENCY', + 'S3DLIO_CHUNK_SIZE', + 'S3DLIO_H2C', + 'S3DLIO_H2_ADAPTIVE_WINDOW', + 'S3DLIO_H2_STREAM_WINDOW_MB', + 'S3DLIO_H2_CONN_WINDOW_MB', + 'S3DLIO_POOL_MAX_IDLE_PER_HOST', + 'S3DLIO_POOL_IDLE_TIMEOUT_SECS', + 'S3DLIO_PUT_VERIFY', + 'S3DLIO_MPU_PUT_VERIFY', + 'S3DLIO_MULTIPART_THRESHOLD_MB', + 'S3DLIO_RT_THREADS', + # 7 MEDIUM-risk S3DLIO_* vars (per D-04) + 'S3DLIO_MAX_RETRY_ATTEMPTS', + 'S3DLIO_CONNECT_TIMEOUT_SECS', + 'S3DLIO_OPERATION_TIMEOUT_SECS', + 'S3DLIO_PUT_MAX_RETRIES', + 'S3DLIO_PUT_RETRY_DELAY_MS', + 'S3DLIO_MPU_MAX_RETRIES', + 'S3DLIO_MPU_RETRY_DELAY_S', + # 3 missing AWS vars (per D-09) — consumed by boto3/s3dlio, no Python os.environ.get() + 'AWS_SESSION_TOKEN', + 'AWS_DEFAULT_REGION', + 'AWS_S3_ADDRESSING_STYLE', +}) + # ----------------------------------------------------------------------------- # ENV_FALLBACK_* module-level constants — resolved-at-import-time env-var From 0f7a18483f651b8be168a27922379f13c4801a2a Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 14:46:25 -0700 Subject: [PATCH 39/45] feat(07.5-01): add Storage-backend tier to ManPage.md; extend AWS-borrowed - Add ### Storage-backend section with 22-row table anchored on s3dlio v0.9.106 - 15 HIGH-risk vars tagged '[high-risk: alters measured benchmark behavior]' - 7 MEDIUM-risk vars tagged '[medium-risk: affects run stability, not throughput measurement]' - Add 3 new rows to ### AWS-borrowed: AWS_SESSION_TOKEN, AWS_DEFAULT_REGION, AWS_S3_ADDRESSING_STYLE - Update ## ENVIRONMENT intro from 'Six tiers' to 'Seven tiers' including Storage-backend - Add Azure/GCS deferred pointer to s3dlio Environment_Variables.md --- ManPage.md | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/ManPage.md b/ManPage.md index cb0c94d2..b660c031 100644 --- a/ManPage.md +++ b/ManPage.md @@ -931,7 +931,7 @@ Reports which Rules.md IDs are referenced by `@rule(rule_id=...)`-decorated chec ## ENVIRONMENT -This section enumerates every environment variable mlpstorage reads or borrows, grouped by ownership tier. Six tiers: Owned (mlpstorage authors the name), MPI-borrowed / AWS-borrowed / Storage-borrowed (third-party contracts mlpstorage consumes), Diagnostic (mlpstorage internal reads surfaced for troubleshooting), and Internal-write (mlpstorage may overwrite user-set values during execution). An automated sync test (`tests/unit/test_env_var_manpage_sync.py`) asserts this section stays symmetric with the actual `os.environ` reads in the codebase. +This section enumerates every environment variable mlpstorage reads or borrows, grouped by ownership tier. Seven tiers: Owned (mlpstorage authors the name), MPI-borrowed / AWS-borrowed / Storage-borrowed (third-party contracts mlpstorage consumes), Storage-backend (s3dlio Rust layer, not Python code), Diagnostic (mlpstorage internal reads surfaced for troubleshooting), and Internal-write (mlpstorage may overwrite user-set values during execution). An automated sync test (`tests/unit/test_env_var_manpage_sync.py`) asserts this section stays symmetric with the actual `os.environ` reads in the codebase. ### Owned @@ -963,6 +963,9 @@ This section enumerates every environment variable mlpstorage reads or borrows, | `AWS_REGION` | `mlpstorage_py/storage_config.py` | `'us-east-1'` | AWS region for S3 endpoint selection; defaults to `us-east-1` when unset. | | `AWS_CA_BUNDLE` | `mlpstorage_py/storage_config.py` | [not set] | Path to a custom CA bundle for TLS verification; unset uses the system default. | | `AWS_ENDPOINT_URL` | `mlpstorage_py/storage_config.py`; `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py:168` | [not set] | Custom S3-compatible endpoint URL. `[cross-ref → internal-write]` — also written at `s3dlio_writer.py:168` during multi-endpoint selection; see Internal-write cross-refs below. | +| `AWS_SESSION_TOKEN` | boto3/s3dlio via storage backend init | [not set] | Temporary STS session token; consumed by boto3/s3dlio via storage backend init. | +| `AWS_DEFAULT_REGION` | boto3/s3dlio via storage backend init | [not set] | Fallback region when `AWS_REGION` is unset; consumed by boto3/s3dlio. | +| `AWS_S3_ADDRESSING_STYLE` | boto3/s3dlio via storage backend init | [not set] | Controls virtual-hosted vs. path-style S3 addressing; consumed by boto3/s3dlio. | ### Storage-borrowed @@ -977,6 +980,35 @@ This section enumerates every environment variable mlpstorage reads or borrows, | `S3_ENDPOINT_FILE` | `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py` | [not set] | Path to a file containing S3 endpoint URIs; s3dlio endpoint fallback chain per D-08. | | `S3_ENDPOINT` | `mlpstorage_py/checkpointing/storage_writers/s3dlio_writer.py` | [not set] | Single S3 endpoint URI; lowest-priority entry in the s3dlio endpoint fallback chain per D-08. | +### Storage-backend + +The following table is anchored on s3dlio v0.9.106. The defaults for `S3DLIO_PUT_VERIFY` and `S3DLIO_MPU_PUT_VERIFY` changed in this release (from `true` to `false`); behavior may differ on earlier versions. For cloud-specific credential and endpoint env vars consumed by s3dlio for Azure or GCS backends, see s3dlio's [Environment_Variables.md](https://github.com/mlcommons/s3dlio/blob/main/docs/Environment_Variables.md). + +| Env var | Read by | Default when unset | Notes | +|---|---|---|---| +| `S3DLIO_SKIP_HEAD` | s3dlio Rust binary (not mlpstorage_py) | `false` | Skips HEAD requests for object existence checks. [high-risk: alters measured benchmark behavior] Enabling this removes per-read existence verification; can significantly increase throughput by reducing API call overhead. | +| `S3DLIO_ENABLE_RANGE_OPTIMIZATION` | s3dlio Rust binary (not mlpstorage_py) | `false` | Enables multi-range read coalescing. [high-risk: alters measured benchmark behavior] Can substantially change read throughput for workloads with many small reads. | +| `S3DLIO_RANGE_THRESHOLD_MB` | s3dlio Rust binary (not mlpstorage_py) | `8` | Minimum size threshold (MB) for range-read coalescing. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_RANGE_CONCURRENCY` | s3dlio Rust binary (not mlpstorage_py) | `4` | Number of concurrent range-read sub-requests. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_CHUNK_SIZE` | s3dlio Rust binary (not mlpstorage_py) | `8388608` | Read/write chunk size in bytes. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_H2C` | s3dlio Rust binary (not mlpstorage_py) | `false` | Enables HTTP/2 cleartext (H2C) transport. [high-risk: alters measured benchmark behavior] Switching transport protocol changes connection multiplexing and can alter throughput substantially. | +| `S3DLIO_H2_ADAPTIVE_WINDOW` | s3dlio Rust binary (not mlpstorage_py) | `false` | Enables HTTP/2 adaptive flow-control window. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_H2_STREAM_WINDOW_MB` | s3dlio Rust binary (not mlpstorage_py) | `1` | HTTP/2 per-stream flow-control window size (MB). [high-risk: alters measured benchmark behavior] | +| `S3DLIO_H2_CONN_WINDOW_MB` | s3dlio Rust binary (not mlpstorage_py) | `1` | HTTP/2 per-connection flow-control window size (MB). [high-risk: alters measured benchmark behavior] | +| `S3DLIO_POOL_MAX_IDLE_PER_HOST` | s3dlio Rust binary (not mlpstorage_py) | `10` | Maximum idle connections per host in the connection pool. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_POOL_IDLE_TIMEOUT_SECS` | s3dlio Rust binary (not mlpstorage_py) | `90` | Time (seconds) before idle connections are evicted. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_PUT_VERIFY` | s3dlio Rust binary (not mlpstorage_py) | `false` | Enables checksum verification on PUT operations. Changed from `true` to `false` in v0.9.106. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_MPU_PUT_VERIFY` | s3dlio Rust binary (not mlpstorage_py) | `false` | Enables checksum verification on multipart PUT operations. Changed from `true` to `false` in v0.9.106. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_MULTIPART_THRESHOLD_MB` | s3dlio Rust binary (not mlpstorage_py) | `64` | Object size threshold (MB) above which multipart upload is used. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_RT_THREADS` | s3dlio Rust binary (not mlpstorage_py) | `1` | Number of Tokio runtime threads. [high-risk: alters measured benchmark behavior] | +| `S3DLIO_MAX_RETRY_ATTEMPTS` | s3dlio Rust binary (not mlpstorage_py) | `3` | Maximum number of retry attempts on transient errors. [medium-risk: affects run stability, not throughput measurement] | +| `S3DLIO_CONNECT_TIMEOUT_SECS` | s3dlio Rust binary (not mlpstorage_py) | `30` | TCP connection timeout in seconds. [medium-risk: affects run stability, not throughput measurement] | +| `S3DLIO_OPERATION_TIMEOUT_SECS` | s3dlio Rust binary (not mlpstorage_py) | `300` | Per-operation timeout in seconds. [medium-risk: affects run stability, not throughput measurement] | +| `S3DLIO_PUT_MAX_RETRIES` | s3dlio Rust binary (not mlpstorage_py) | `3` | Maximum retries for PUT operations. [medium-risk: affects run stability, not throughput measurement] | +| `S3DLIO_PUT_RETRY_DELAY_MS` | s3dlio Rust binary (not mlpstorage_py) | `100` | Delay (ms) between PUT retries. [medium-risk: affects run stability, not throughput measurement] | +| `S3DLIO_MPU_MAX_RETRIES` | s3dlio Rust binary (not mlpstorage_py) | `3` | Maximum retries for multipart upload operations. [medium-risk: affects run stability, not throughput measurement] | +| `S3DLIO_MPU_RETRY_DELAY_S` | s3dlio Rust binary (not mlpstorage_py) | `1` | Delay (seconds) between multipart upload retries. [medium-risk: affects run stability, not throughput measurement] | + ### Diagnostic No environment variables are diagnostic-only under the current inventory (2026-07-04). This tier header is preserved so future diagnostic env vars have a home; the sync test in `tests/unit/test_env_var_manpage_sync.py` allows this tier to be empty. From cbe0728e9c6066e6284f8fb09a6fde8e91c40b2b Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 14:48:10 -0700 Subject: [PATCH 40/45] fix(07.5-01): update sync test to recognize 'storage-backend' as 7th canonical tier - Add 'storage-backend' to tier_canonical dict in _parse_manpage_env_vars so S3DLIO_* rows are parsed from ManPage tables (not silently skipped) - Update test_every_tier_value_is_one_of_six_canonical_strings valid_tiers set to include 'storage-backend' (Phase 7.5 D-01) - TestEnvVarInventorySync.test_code_reads_equal_manpage_mentions_modulo_allowlist remains a known failure until Plan 02 adds MANPAGE_STORAGE_BACKEND_ENV_VARS subtraction --- tests/unit/test_env_var_manpage_sync.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_env_var_manpage_sync.py b/tests/unit/test_env_var_manpage_sync.py index c8e7c832..adc49bc6 100644 --- a/tests/unit/test_env_var_manpage_sync.py +++ b/tests/unit/test_env_var_manpage_sync.py @@ -220,6 +220,7 @@ def _parse_manpage_env_vars(manpage_path: pathlib.Path) -> dict[str, str]: "mpi-borrowed": "mpi-borrowed", "aws-borrowed": "aws-borrowed", "storage-borrowed": "storage-borrowed", + "storage-backend": "storage-backend", # Phase 7.5 D-01: s3dlio Rust-layer vars "diagnostic": "diagnostic", "internal-write": "internal-write", } @@ -393,11 +394,11 @@ def test_every_manpage_env_var_matches_declared_tier(self): ) def test_every_tier_value_is_one_of_six_canonical_strings(self): - """MANPAGE_ENV_VAR_TIERS values must be one of the six canonical tier strings (D-07). + """MANPAGE_ENV_VAR_TIERS values must be one of the seven canonical tier strings (Phase 7.5 D-01). - The six valid values are: + The seven valid values are: 'owned', 'mpi-borrowed', 'aws-borrowed', 'storage-borrowed', - 'diagnostic', 'internal-write' + 'storage-backend', 'diagnostic', 'internal-write' Any spelling variation (extra whitespace, wrong capitalisation, unknown tier) fails this test. @@ -407,6 +408,7 @@ def test_every_tier_value_is_one_of_six_canonical_strings(self): "mpi-borrowed", "aws-borrowed", "storage-borrowed", + "storage-backend", # Phase 7.5 D-01: s3dlio Rust-layer vars "diagnostic", "internal-write", } From 336fa2f8aef76dc28a889a04d5ca41eed7c90a32 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 14:53:02 -0700 Subject: [PATCH 41/45] feat(07.5-02): add _check_storage_backend_env() to Benchmark base class - Import _S3DLIO_HIGH_RISK_ENV_VARS from mlpstorage_py.config - Add private method Benchmark._check_storage_backend_env() with D-08 verbatim warning template - Guard on STORAGE_LIBRARY='s3dlio' to avoid false positives in training-only environments (D-07) - Call from _pre_execution_gate() after CAP-03 probe (Phase 7.5 D-07) --- mlpstorage_py/benchmarks/base.py | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/mlpstorage_py/benchmarks/base.py b/mlpstorage_py/benchmarks/base.py index 911b6d8f..1eac8637 100755 --- a/mlpstorage_py/benchmarks/base.py +++ b/mlpstorage_py/benchmarks/base.py @@ -46,7 +46,13 @@ def _run(self): from functools import wraps -from mlpstorage_py.config import PARAM_VALIDATION, DATETIME_STR, MLPS_DEBUG, EXEC_TYPE +from mlpstorage_py.config import ( + PARAM_VALIDATION, + DATETIME_STR, + MLPS_DEBUG, + EXEC_TYPE, + _S3DLIO_HIGH_RISK_ENV_VARS, +) from mlpstorage_py.errors import ConfigurationError, ErrorCode, FileSystemError from mlpstorage_py.run_directory import ( DEFAULT_COLLISION_BUMP_BUDGET, @@ -991,6 +997,30 @@ def _fs_separation_paths(self) -> Optional[tuple]: f"_fs_separation_paths() for CAP-03" ) + def _check_storage_backend_env(self) -> None: + """Warn when HIGH-risk s3dlio env vars are set (Phase 7.5 D-07, D-08). + + Checks ``os.environ`` for entries in ``_S3DLIO_HIGH_RISK_ENV_VARS`` + and emits a WARNING for each one found, but only when the active + storage library is s3dlio (``STORAGE_LIBRARY == 's3dlio'``). + + Rationale for the STORAGE_LIBRARY guard (D-07): training-only + deployments that never use s3dlio should not see s3dlio warnings. + The guard avoids false positives without requiring per-var allowlists. + + The warning uses the D-08 verbatim template so that automated tests + (test_d08_warning_prefix_in_benchmark_source) can lock the wording. + """ + if os.environ.get('STORAGE_LIBRARY') != 's3dlio': + return + for var_name in sorted(_S3DLIO_HIGH_RISK_ENV_VARS): + if var_name in os.environ: + self.logger.warning( + f"s3dlio env var '{var_name}' is set — this may alter measured benchmark results. " + "Declare it in your config or submit as OPEN category. " + "See ManPage §ENVIRONMENT → Storage-backend for impact." + ) + def _pre_execution_gate(self) -> None: """Run all pre-execution capacity/environment gates (CAP-01 in Slice 3; CAP-02 shared-FS verification is appended in Slice 4 of @@ -1066,6 +1096,8 @@ def _pre_execution_gate(self) -> None: # design decision D-601-1). --skip-fs-separation-gate bypasses the # raise but still writes the sidecar so the validator gets telemetry. self._run_fs_separation_probe() + # Phase 7.5 D-07: warn when HIGH-risk s3dlio env vars are set. + self._check_storage_backend_env() def _run_fs_separation_probe(self) -> None: """Run the CAP-03 FS-separation probe and write the sidecar. From 36a4eab547d2703692dd291997dfd219e4bddc92 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 14:53:51 -0700 Subject: [PATCH 42/45] feat(07.5-02): extend sync test with MANPAGE_STORAGE_BACKEND_ENV_VARS carve-out and TestStorageBackendEnvVars - Import MANPAGE_STORAGE_BACKEND_ENV_VARS from config - Subtract MANPAGE_STORAGE_BACKEND_ENV_VARS from extra_in_docs so storage-backend vars don't fail the symmetric-difference check - Add TestStorageBackendEnvVars class with 3 test methods: backend vars in ManPage, disjoint from allowlist, D-08 verbatim prefix lock --- tests/unit/test_env_var_manpage_sync.py | 74 ++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_env_var_manpage_sync.py b/tests/unit/test_env_var_manpage_sync.py index adc49bc6..d78a25aa 100644 --- a/tests/unit/test_env_var_manpage_sync.py +++ b/tests/unit/test_env_var_manpage_sync.py @@ -35,7 +35,11 @@ import pathlib import pytest -from mlpstorage_py.config import _MANPAGE_SYNC_ALLOWLIST, MANPAGE_ENV_VAR_TIERS +from mlpstorage_py.config import ( + _MANPAGE_SYNC_ALLOWLIST, + MANPAGE_ENV_VAR_TIERS, + MANPAGE_STORAGE_BACKEND_ENV_VARS, +) # --------------------------------------------------------------------------- @@ -268,7 +272,7 @@ def test_code_reads_equal_manpage_mentions_modulo_allowlist(self): docs_mentions = set(_parse_manpage_env_vars(_MANPAGE).keys()) missing_in_docs = code_reads - _MANPAGE_SYNC_ALLOWLIST - docs_mentions - extra_in_docs = docs_mentions - _MANPAGE_SYNC_ALLOWLIST - code_reads + extra_in_docs = docs_mentions - _MANPAGE_SYNC_ALLOWLIST - MANPAGE_STORAGE_BACKEND_ENV_VARS - code_reads assert missing_in_docs == set() and extra_in_docs == set(), ( "Env-var inventory out of sync (SC-3 / DOC-05).\n\n" @@ -564,3 +568,69 @@ def test_no_legacy_env_var_names_in_docs_dir(self): + "\n".join(offenders) + "\n\nFix: rename to MLPSTORAGE_* equivalents throughout docs/." ) + + +# =========================================================================== +# Test Class 5: Storage-backend env var carve-out (Phase 7.5 D-02, D-08) +# =========================================================================== + + +class TestStorageBackendEnvVars: + """Phase 7.5 D-02: MANPAGE_STORAGE_BACKEND_ENV_VARS carve-out assertions. + + Verifies that: + 1. Every entry in MANPAGE_STORAGE_BACKEND_ENV_VARS appears in ManPage.md + under the tier declared in MANPAGE_ENV_VAR_TIERS. + 2. MANPAGE_STORAGE_BACKEND_ENV_VARS and _MANPAGE_SYNC_ALLOWLIST are disjoint + (backend vars are DOCUMENTED; allowlist vars are NOT). + 3. The D-08 verbatim warning prefix appears in benchmark source. + """ + + def test_backend_vars_appear_in_manpage(self): + """Every entry in MANPAGE_STORAGE_BACKEND_ENV_VARS is in ManPage under its declared tier.""" + manpage_tiers = _parse_manpage_env_vars(_MANPAGE) + missing = [] + wrong_tier = [] + for var_name in MANPAGE_STORAGE_BACKEND_ENV_VARS: + declared_tier = MANPAGE_ENV_VAR_TIERS.get(var_name) + actual_tier = manpage_tiers.get(var_name) + if actual_tier is None: + missing.append(var_name) + elif declared_tier is not None and actual_tier != declared_tier: + wrong_tier.append( + f" {var_name!r}: expected={declared_tier!r}, got={actual_tier!r}" + ) + assert not missing and not wrong_tier, ( + "MANPAGE_STORAGE_BACKEND_ENV_VARS integrity failure (Phase 7.5 D-02):\n" + + (f"\nNot found in ManPage:\n {sorted(missing)}" if missing else "") + + ("\nWrong tier in ManPage:\n" + "\n".join(wrong_tier) if wrong_tier else "") + ) + + def test_backend_vars_disjoint_from_allowlist(self): + """MANPAGE_STORAGE_BACKEND_ENV_VARS must not overlap with _MANPAGE_SYNC_ALLOWLIST. + + Backend vars are DOCUMENTED in ManPage tier tables (they have a tier entry). + Allowlist vars are NOT documented (they are intentionally excluded). + Overlap means a var is simultaneously 'documented but not read by Python' + AND 'exempt from documentation' — a contradiction. + """ + overlap = MANPAGE_STORAGE_BACKEND_ENV_VARS & _MANPAGE_SYNC_ALLOWLIST + assert not overlap, ( + "MANPAGE_STORAGE_BACKEND_ENV_VARS and _MANPAGE_SYNC_ALLOWLIST overlap " + f"(Phase 7.5 D-02/D-05 disjoint invariant violated):\n {sorted(overlap)}\n\n" + "Backend vars are documented; allowlist vars are not. They must be disjoint." + ) + + def test_d08_warning_prefix_in_benchmark_source(self): + """D-08 verbatim warning prefix must appear in mlpstorage_py/benchmarks/base.py. + + Locks the warn-and-continue check added by Phase 7.5 Plan 02 Task 1. + If the method is removed or the string is paraphrased, this test fails. + """ + base_src = (_MLPSTORAGE_PY / "benchmarks" / "base.py").read_text() + assert "s3dlio env var '" in base_src, ( + "D-08 verbatim warning prefix \"s3dlio env var '\" not found in " + "mlpstorage_py/benchmarks/base.py.\n\n" + "Fix: ensure _check_storage_backend_env() in base.py uses the verbatim " + "template from Phase 7.5 D-08 (do not paraphrase)." + ) From f1258f3f13563927ed9c7eb679677aae6f7dc0e5 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 14:54:35 -0700 Subject: [PATCH 43/45] feat(07.5-02): add D-14 storageBackendEnvVarDeclaration rule to Rules.md - Add rule 1.2 storageBackendEnvVarDeclaration with D-14 verbatim wording - Requires OPEN category for runs with undeclared storage-backend env vars - Adds Azure/GCS pointer to s3dlio Environment_Variables.md --- Rules.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Rules.md b/Rules.md index f7aed0d5..44d7ee22 100644 --- a/Rules.md +++ b/Rules.md @@ -54,6 +54,10 @@ The `mlpstorage` tool must be used to run the benchmarks, submitters are not all 1.1. **mlpstorageGeneratesHierarchy** -- The `mlpstorage` command must obtain (somehow) the pathname of the output file directory hierarchy and directly create and/or append to the files within that hierarchy to successively build out the submission folder. We don't want the submitter to manually create anything in that hierarchy except for the SystemDescription.* files (if we can help it). +1.2. **storageBackendEnvVarDeclaration** -- Submitters MUST NOT set any documented s3dlio-side env var (§ENVIRONMENT → Storage-backend in ManPage.md) during the timed run unless the run's configuration file explicitly declares it. Results from runs with undeclared storage-backend env vars MUST be submitted as OPEN category. mlpstorage warns at run start when HIGH-risk vars are detected. + +For cloud-specific credential and endpoint env vars consumed by s3dlio for Azure or GCS backends, see s3dlio's [Environment_Variables.md](https://github.com/mlcommons/s3dlio/blob/main/docs/Environment_Variables.md). + # 2. Core/Common Rules for All Submissions ## 2.1. Core/Common POSIX API Rules From e12e813a4fc56a85b86410fa1ed0e389a20859f6 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Sun, 5 Jul 2026 15:36:23 -0700 Subject: [PATCH 44/45] fix: align _enumerate_on_disk_model_dirs to return /run/ for training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _model_group_folder returns /run/ for training (Rules.md 2.1.16), but _enumerate_on_disk_model_dirs was returning /. The mismatch caused _emit_empty_model_dirs to treat / as an unaccounted dir and write a stray results.json there. Now training dirs enumerate to /run/. Also update test_reportgen_output_shape expected paths to match, and fix MLPERF_SYSTEMNAME → MLPSTORAGE_SYSTEMNAME typo in Rules.md. --- Rules.md | 2 +- mlpstorage_py/report_generator.py | 8 ++++++++ tests/unit/test_reportgen_output_shape.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Rules.md b/Rules.md index 44d7ee22..1521f723 100644 --- a/Rules.md +++ b/Rules.md @@ -674,7 +674,7 @@ is better), and the **aggregate Storage Throughput in tokens/s**. A run is launched with `mlpstorage kvcache run [OPTIONS]` (the division is the first positional, post-PR #412 modal CLI). The results directory must first be initialised once with `mlpstorage init -`, and every run requires `--systemname` (or the `MLPERF_SYSTEMNAME` +`, and every run requires `--systemname` (or the `MLPSTORAGE_SYSTEMNAME` environment variable). The command executes **all three Options sequentially**, each repeated `trials` times, by prefixing `mpirun` to `mlperf_wrapper.py`. The Option's parameters are built by `mlpstorage` (`_build_option_kvcache_args`) and diff --git a/mlpstorage_py/report_generator.py b/mlpstorage_py/report_generator.py index bc9ed68f..61a18e44 100755 --- a/mlpstorage_py/report_generator.py +++ b/mlpstorage_py/report_generator.py @@ -613,6 +613,14 @@ def _enumerate_on_disk_model_dirs(self) -> List[str]: for index_dir in model_dir.iterdir(): if index_dir.is_dir(): found.add(str(index_dir.resolve())) + elif bt_dir.name == 'training': + # Rules.md 2.1.16: model-group folder is + # /run/, not / — must match + # _model_group_folder so _emit_empty_model_dirs + # doesn't write a stray file at /. + run_dir = model_dir / 'run' + if run_dir.is_dir(): + found.add(str(run_dir.resolve())) else: found.add(str(model_dir.resolve())) except (OSError, PermissionError) as e: diff --git a/tests/unit/test_reportgen_output_shape.py b/tests/unit/test_reportgen_output_shape.py index bcb5b292..5c03fb20 100644 --- a/tests/unit/test_reportgen_output_shape.py +++ b/tests/unit/test_reportgen_output_shape.py @@ -446,11 +446,11 @@ def test_multi_orgname_produces_single_top_level_file_with_mixed_rows( # D-09: per-org per-model files exist at their canonical paths. acme_per_model_json = ( dest / "closed" / "acme" / "results" / "system-a" - / "training" / "unet3d" / "results.json" + / "training" / "unet3d" / "run" / "results.json" ) beta_per_model_json = ( dest / "closed" / "beta_corp" / "results" / "system-b" - / "training" / "unet3d" / "results.json" + / "training" / "unet3d" / "run" / "results.json" ) assert acme_per_model_json.exists(), ( f"D-09 violated: expected per-model rollup at {acme_per_model_json}" From 43294f7417a7e9be29090a011894128d32509e87 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Tue, 7 Jul 2026 13:26:25 -0700 Subject: [PATCH 45/45] =?UTF-8?q?fix:=20remove=20=C2=A72.1.28=20aggregateR?= =?UTF-8?q?esultsFile=20from=20Rules.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Descriptive-only clause with no enforceable checker contract; not present in origin/main. Resolves rules-coverage CI failure (unmapped rule 2.1.28). --- Rules.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Rules.md b/Rules.md index 1521f723..e0172ea4 100644 --- a/Rules.md +++ b/Rules.md @@ -341,10 +341,6 @@ root_folder (or any name you prefer) ├──system-name-2.yaml └──system-name-2.pdf ``` -2.1.28. **aggregateResultsFile** -- Reportgen assembles one top-level results file per invocation at `/results.{csv,json}` by walking each per-workload directory in the tree, computing that workload's aggregated row, writing the per-model `<...>//results.{csv,json}` files first, and then collecting those per-model rows into the top-level file (bottom-up build; the top-level file is a collection step, not an aggregation step). Each row in the top-level file represents ONE workload (submitter parlance: "run") of any benchmark type — training, checkpointing, vdb, or kvcache — with no per-invocation raw rows and no per-row discriminator column. Every reportgen invocation rebuilds these files from scratch, so deleted run directories disappear from the next report; empty model directories still emit a header-only CSV and an empty-list JSON. - -The row layout has a fixed 6-column prefix in this exact order: `category`, `orgname`, `systemname`, `benchmark_type`, `model`, `accelerator`. After the prefix, columns are grouped by benchmark type in the fixed order training → checkpointing → vdb → kvcache, with every column in a group carrying that group's prefix (`train_`, `checkpoint_`, `vdb_`, `kvcache_`). Within each group, columns are alphabetical. The trailing column is always `issues` — variable-length text carrying verbatim `Result.issues` messages joined by `; ` per §2.1.17 and §2.1.23 validation contracts. The `category` column takes one of four values: `closed`, `open`, `whatif`, or `INVALID`; the `INVALID` value is emitted when a workload violates the rules-strict counts described in §2.1.17 (training must be exactly 6 invocations, 1 warmup + 5 real) and §2.1.23 (checkpointing must have exactly 10 checkpoint operations per invocation). `whatif` rows are simulation output and skip the rules-strict INVALID gates entirely. - 2.29. **dlioLog** -- Since the "dlio_log" subdirectory has a similar structure in all cases, it is describe pictorially just below: ``` └── YYYYMMDD_HHmmss