diff --git a/mlpstorage_py/submission_checker/checks/helpers.py b/mlpstorage_py/submission_checker/checks/helpers.py index 35884c57..2468ec89 100644 --- a/mlpstorage_py/submission_checker/checks/helpers.py +++ b/mlpstorage_py/submission_checker/checks/helpers.py @@ -38,6 +38,10 @@ CodeImageError, MissingHashFile, MalformedHashFile, + PointerMalformed, + _read_pointer, + _pool_dir_name, + _read_hash_file, ) @@ -347,6 +351,59 @@ def _check_code_image_layered( return valid +# --------------------------------------------------------------------------- +# resolve_run_pool_image (Phase 8 D-89, CHECK-05) +# --------------------------------------------------------------------------- + +def resolve_run_pool_image( + run_leaf: Path, + results_dir: Path, + orgname: str, + log, +) -> tuple[Path, dict]: + """Resolve run leaf → pool image path + hash data. + + Steps 1-4 of the D-89 CHECK-05 flow. Steps 5-6 (REFERENCE_CHECKSUMS lookup + and upstream-identity comparison) are caller responsibility. + + Raises FileNotFoundError if .mlps-code-image is absent, PointerMalformed if + malformed, MissingHashFile / MalformedHashFile if .code-hash.json is absent + or invalid. + + Args: + run_leaf: Path to the per-timestamp run directory that holds the + ``.mlps-code-image`` pointer file. + results_dir: The top-level submission root (``args.input``). The pool + lives at ``//code-/`` per D-89/D-62. + orgname: Organization name (directory name under the top-level pool + root, e.g. ``"acme"``). + log: Logger object passed through to ``_read_pointer`` and + ``_read_hash_file``. + + Returns: + ``(pool_image_path, hash_data)`` where ``pool_image_path`` is the + resolved ``Path`` to the pool image directory and ``hash_data`` is the + parsed ``.code-hash.json`` dict (keys: ``hash``, ``algorithm``, + ``captured_at``, ``mlpstorage_version``, ``git_sha``). + + Raises: + FileNotFoundError: ``.mlps-code-image`` absent in ``run_leaf``. + PointerMalformed: File present but not a valid ``md5-tree-v2:``. + MissingHashFile: ``.code-hash.json`` absent in the resolved pool image. + MalformedHashFile: ``.code-hash.json`` present but invalid. + """ + # Step 1-2: read pointer; propagate FileNotFoundError and PointerMalformed. + _alg, full_hash = _read_pointer(run_leaf, log) + + # Step 3: resolve pool image path. + pool_image_path = results_dir / orgname / _pool_dir_name(full_hash) + + # Step 4: read hash file; propagate MissingHashFile and MalformedHashFile. + hash_data = _read_hash_file(pool_image_path, log) + + return pool_image_path, hash_data + + # --------------------------------------------------------------------------- # _pair_checkpoint_runs # --------------------------------------------------------------------------- diff --git a/mlpstorage_py/submission_checker/checks/training_checks.py b/mlpstorage_py/submission_checker/checks/training_checks.py index b15b3b5f..8cb7bc26 100644 --- a/mlpstorage_py/submission_checker/checks/training_checks.py +++ b/mlpstorage_py/submission_checker/checks/training_checks.py @@ -5,11 +5,22 @@ from ..dlio_summary_helpers import cluster_total_host_memory_gb from ..loader import SubmissionLogs from ..rule_registry import rule +from pathlib import Path + from .helpers import ( _check_filesystem_separation, _check_code_image_layered, read_fs_separation_sidecar, + resolve_run_pool_image, +) +from ..tools.code_image import ( + verify_image_self_consistent, + PointerMalformed, + MissingHashFile, + MalformedHashFile, ) +from ..tools.code_checksum import compute_code_tree_md5 +from ..constants import REFERENCE_CHECKSUMS # Shared with the in-process verifier (mlpstorage_py.rules.run_checkers.training) # so both checkers stay in lockstep about which dotted-keys the mlpstorage @@ -681,32 +692,127 @@ def node_capability_consistency_check(self): @rule("3.6.1", "trainingClosedSubmissionChecksum") def closed_submission_checksum(self): - """For CLOSED submissions, verify code directory MD5 checksum. + """For CLOSED training submissions, verify per-image code checksum (D-89, CHECK-05). (Rules.md 3.6.1) - Phase 4 CD-04: delegates to the shared - ``helpers._check_code_image_layered`` helper so the §3.6.1 and §5.6.1 - rules enforce an identical layered model (self-consistency + - upstream-identity) without duplicating the implementation across - check classes. STRUCT-06 (§2.1.6) keeps its own inline implementation - because it has additional surrounding logic (per-leaf walker, the - ``expected is None`` warning) that does not belong in the helper. - - Walk-up: ``self.path`` is the per-leaf training path - (``/closed//results//training/``). The - CLOSED ``code/`` lives at ``/closed//code/``, four - levels above ``self.path`` (model → type → system → results → - ````). Missing ``code/`` is NOT logged here — STRUCT-06 - already owns the VALS-01 missing-code/ violation under §2.1.6, so - re-firing under §3.6.1 would double-count. + Phase 8 Plan 02: replaces the legacy code/-walk with the full D-89 + per-image pool-walk. For each run leaf, reads ``.mlps-code-image`` to + resolve the pool image, runs self-consistency verification, then looks up + ``REFERENCE_CHECKSUMS[mlpstorage_version]`` for upstream-identity check. + + Walk-up: ``self.path`` is + ``/closed//results//training/``. + Run leaves live at ``self.run_path//``. From a datetime leaf: + parents[5] → ```` (submitter_path) + parents[7] → ```` (results_dir / pool root parent) """ - # TODO(Phase8-Plan2): replaced in Plan 08-02 with per-image pool-image - # lookup (D-89). The legacy code/ walk and get_reference_checksum() call - # are removed; Plan 08-02 implements the full CHECK-05 flow using - # _read_pointer → pool image resolution → verify_image_self_consistent - # → REFERENCE_CHECKSUMS[mlpstorage_version] per-image lookup. - return True # TODO(Phase8-Plan2): replaced in Plan 08-02 + if self.mode != "training": + return True + if self.submissions_logs.loader_metadata.division != "closed": + return True + + valid = True + # D-87: emit the "unknown version" warning at most once per pool image. + if not hasattr(self, "_warned_pool_images"): + self._warned_pool_images: set[str] = set() + + for _summary, _metadata, ts in self.submissions_logs.run_files: + run_leaf = Path(self.run_path) / ts + + # Determine submitter_path and root_path by walking up from run_leaf. + # run_leaf depth from root: /closed//results//training//run/ + # parents[5] = parents[7] = + run_leaf_path = Path(run_leaf) + try: + submitter_path = run_leaf_path.parents[5] + root_path = run_leaf_path.parents[7] + except IndexError: + self.log_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(run_leaf), + "unexpected directory depth; cannot resolve pool root", + ) + valid = False + continue + + orgname = submitter_path.name + + # Steps 1-4 (D-89): resolve run leaf → pool image + hash data. + try: + pool_image_path, hash_data = resolve_run_pool_image( + run_leaf_path, root_path, orgname, self.log, + ) + except FileNotFoundError: + self.log_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(run_leaf), + "run leaf %s has no .mlps-code-image pointer.", + run_leaf, + ) + valid = False + continue + except PointerMalformed as e: + self.log_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(run_leaf), + "%s", str(e), + ) + valid = False + continue + except (MissingHashFile, MalformedHashFile) as e: + self.log_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(run_leaf), + "%s", str(e), + ) + valid = False + continue + + # Step 5 (D-89): self-consistency check on pool image. + try: + self_ok = verify_image_self_consistent(pool_image_path, self.log) + except Exception as e: + self.log_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(pool_image_path), + "%s", str(e), + ) + valid = False + continue + if not self_ok: + self.log_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(pool_image_path), + "pool image %s: self-consistency check failed", + pool_image_path, + ) + valid = False + continue + + # Step 6 (D-89): version-keyed upstream-identity check (D-86/D-87). + mlpstorage_version = hash_data.get("mlpstorage_version") + expected = REFERENCE_CHECKSUMS.get(mlpstorage_version) + + if expected is None: + # D-87: warn once per pool image, then pass. + pool_key = str(pool_image_path) + if pool_key not in self._warned_pool_images: + self._warned_pool_images.add(pool_key) + self.warn_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(pool_image_path), + "mlpstorage_version %s not in REFERENCE_CHECKSUMS; " + "upstream-identity check skipped (self-consistency still ran).", + mlpstorage_version, + ) + continue + + # Step 7: compute tree MD5 and compare against reference. + digest = compute_code_tree_md5(str(pool_image_path), self.log) + if digest != expected: + self.log_violation( + "3.6.1", "trainingClosedSubmissionChecksum", str(pool_image_path), + "pool image %s: MD5 %s does not match " + "REFERENCE_CHECKSUMS[%s] = %s", + pool_image_path, digest, mlpstorage_version, expected, + ) + valid = False + + return valid @rule("3.6.2", "trainingClosedSubmissionParameters") def closed_submission_parameters(self): diff --git a/mlpstorage_py/submission_checker/checks/vdb_checks.py b/mlpstorage_py/submission_checker/checks/vdb_checks.py index c9348f6a..30c07338 100644 --- a/mlpstorage_py/submission_checker/checks/vdb_checks.py +++ b/mlpstorage_py/submission_checker/checks/vdb_checks.py @@ -28,6 +28,7 @@ """ import os +from pathlib import Path from .base import BaseCheck from ..configuration.configuration import Config @@ -37,7 +38,16 @@ _check_code_image_layered, _check_filesystem_separation, read_fs_separation_sidecar, + resolve_run_pool_image, ) +from ..tools.code_image import ( + verify_image_self_consistent, + PointerMalformed, + MissingHashFile, + MalformedHashFile, +) +from ..tools.code_checksum import compute_code_tree_md5 +from ..constants import REFERENCE_CHECKSUMS from mlpstorage_py.config import VDB_INDEX_TYPES_CLOSED @@ -749,29 +759,127 @@ def vdb_object_storage_backend(self): @rule("5.6.1", "vdbClosedSubmissionChecksum") def vdb_closed_submission_checksum(self): - """For CLOSED submissions, verify the code-image self-consistency + - upstream-identity via the shared layered helper. (Rules.md 5.6.1) - - Phase 4 D-06 / CD-04: delegates to - ``helpers._check_code_image_layered`` — the SAME helper - ``TrainingCheck.3.6.1`` calls — so the layered model is implemented - once and attributed under the caller's rule ID/name. - - Walk-up: ``self.path`` is the per-leaf vdb path - (``/closed//results//vector_database/``). - The CLOSED ``code/`` lives at ``/closed//code/``, - four levels above ``self.path`` (DisplayIndex → vector_database → system - → results → ````). - - Missing ``code/`` is NOT logged here — STRUCT-06 (§2.1.6) owns the - VALS-01 missing-code/ violation; re-firing here would double-count. + """For CLOSED VDB submissions, verify per-image code checksum (D-89, CHECK-05). + + (Rules.md 5.6.1) + + Phase 8 Plan 02: replaces the legacy code/-walk with the full D-89 + per-image pool-walk. For each run leaf, reads ``.mlps-code-image`` to + resolve the pool image, runs self-consistency verification, then looks up + ``REFERENCE_CHECKSUMS[mlpstorage_version]`` for upstream-identity check. + + Walk-up: ``self.path`` is + ``/closed//results//vector_database//``. + Run leaves live at ``self.run_path//``. From a datetime leaf: + parents[6] → ```` (submitter_path) + parents[8] → ```` (results_dir / pool root parent) """ - # TODO(Phase8-Plan2): replaced in Plan 08-02 with per-image pool-image - # lookup (D-89). The legacy code/ walk and get_reference_checksum() call - # are removed; Plan 08-02 implements the full CHECK-05 flow using - # _read_pointer → pool image resolution → verify_image_self_consistent - # → REFERENCE_CHECKSUMS[mlpstorage_version] per-image lookup. - return True # TODO(Phase8-Plan2): replaced in Plan 08-02 + if self.mode != "vector_database": + return True + if self.division != "closed": + return True + + valid = True + # D-87: emit the "unknown version" warning at most once per pool image. + if not hasattr(self, "_warned_pool_images"): + self._warned_pool_images: set[str] = set() + + for _summary, _metadata, ts in self._iter_run_files(): + run_leaf = Path(self.run_path) / ts + + # Determine submitter_path and root_path by walking up from run_leaf. + # run_leaf depth: /closed//results//vector_database///run/ + # parents[6] = parents[8] = + run_leaf_path = Path(run_leaf) + try: + submitter_path = run_leaf_path.parents[6] + root_path = run_leaf_path.parents[8] + except IndexError: + self.log_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(run_leaf), + "unexpected directory depth; cannot resolve pool root", + ) + valid = False + continue + + orgname = submitter_path.name + + # Steps 1-4 (D-89): resolve run leaf → pool image + hash data. + try: + pool_image_path, hash_data = resolve_run_pool_image( + run_leaf_path, root_path, orgname, self.log, + ) + except FileNotFoundError: + self.log_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(run_leaf), + "run leaf %s has no .mlps-code-image pointer.", + run_leaf, + ) + valid = False + continue + except PointerMalformed as e: + self.log_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(run_leaf), + "%s", str(e), + ) + valid = False + continue + except (MissingHashFile, MalformedHashFile) as e: + self.log_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(run_leaf), + "%s", str(e), + ) + valid = False + continue + + # Step 5 (D-89): self-consistency check on pool image. + try: + self_ok = verify_image_self_consistent(pool_image_path, self.log) + except Exception as e: + self.log_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(pool_image_path), + "%s", str(e), + ) + valid = False + continue + if not self_ok: + self.log_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(pool_image_path), + "pool image %s: self-consistency check failed", + pool_image_path, + ) + valid = False + continue + + # Step 6 (D-89): version-keyed upstream-identity check (D-86/D-87). + mlpstorage_version = hash_data.get("mlpstorage_version") + expected = REFERENCE_CHECKSUMS.get(mlpstorage_version) + + if expected is None: + # D-87: warn once per pool image, then pass. + pool_key = str(pool_image_path) + if pool_key not in self._warned_pool_images: + self._warned_pool_images.add(pool_key) + self.warn_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(pool_image_path), + "mlpstorage_version %s not in REFERENCE_CHECKSUMS; " + "upstream-identity check skipped (self-consistency still ran).", + mlpstorage_version, + ) + continue + + # Step 7: compute tree MD5 and compare against reference. + digest = compute_code_tree_md5(str(pool_image_path), self.log) + if digest != expected: + self.log_violation( + "5.6.1", "vdbClosedSubmissionChecksum", str(pool_image_path), + "pool image %s: MD5 %s does not match " + "REFERENCE_CHECKSUMS[%s] = %s", + pool_image_path, digest, mlpstorage_version, expected, + ) + valid = False + + return valid @rule("5.6.2", "vdbClosedDatabaseBackend") def vdb_closed_database_backend(self): diff --git a/tests/integration/test_submission_checker_pool_v11.py b/tests/integration/test_submission_checker_pool_v11.py index a66e15d9..227eef0d 100644 --- a/tests/integration/test_submission_checker_pool_v11.py +++ b/tests/integration/test_submission_checker_pool_v11.py @@ -1,4 +1,6 @@ -"""Integration tests for Phase 8 success criteria (SC-1..SC-4). +"""Integration tests for Phase 8 success criteria (SC-1..SC-5). + +Phase 08-03 Plan 03 Task 2. Exercises the complete submission checker run() function against v1.1 pool layout trees, covering: @@ -7,6 +9,7 @@ SC-2: missing/dangling .mlps-code-image → run() returns 1 with CHECK-01 message SC-3: modified pool image / renamed pool dir → run() returns 1 with CHECK-02 message SC-4: orphan pool image / legacy code/ dir → run() returns 1 with CHECK-03/04 message + SC-5: two-version two-pool-image tree → run() returns 0 (D-86 per-image lookup) Strategy: The run() function also invokes SubmissionStructureCheck, SystemYamlSchemaCheck, DirectoryCheck, and a full TrainingCheck suite (many @@ -16,9 +19,17 @@ - SubmissionStructureCheck and SystemYamlSchemaCheck are monkeypatched to no-op (return True) — they already have their own test coverage. - - MODE_TO_CHECKERS is monkeypatched to an empty checker list so the full - TrainingCheck battery does not run against the minimal v1.1 fixtures. + - MODE_TO_CHECKERS is monkeypatched to use a slimmed PoolAwareTrainingCheck + that only runs closed_submission_checksum (CHECK-05) instead of the full + TrainingCheck battery. This lets SC-5 exercise the real D-89 pool walk + without needing 6-timestamp run leaves, results.json, dlio_config/, etc. - PoolStructureCheck is never monkeypatched — it runs for real in all tests. + +SC-1..SC-4 tests use a minimal v1.1 tree with one run leaf. SC-5 uses a tree +with two run leaves and two pool images (different content → different hashes). + +Refs: 08-CONTEXT.md D-80..D-93, 08-01-SUMMARY.md, 08-02-SUMMARY.md, + 08-03-PLAN.md Task 2. """ from __future__ import annotations @@ -28,8 +39,13 @@ import logging import shutil from pathlib import Path +from unittest.mock import MagicMock + import pytest +from mlpstorage_py.submission_checker.checks.base import BaseCheck +from mlpstorage_py.submission_checker.checks.training_checks import TrainingCheck +from mlpstorage_py.submission_checker.constants import REFERENCE_CHECKSUMS from mlpstorage_py.submission_checker.main import run from mlpstorage_py.submission_checker.tools.code_checksum import compute_code_tree_md5 from mlpstorage_py.submission_checker.tools.code_image import ( @@ -200,6 +216,36 @@ def v11_tree_factory(tmp_path): return _V11TreeFactory(tmp_path) +# --------------------------------------------------------------------------- +# PoolAwareTrainingCheck — slim TrainingCheck for SC-5 +# --------------------------------------------------------------------------- + +class _PoolAwareTrainingCheck(BaseCheck): + """Slim TrainingCheck that only runs closed_submission_checksum (CHECK-05). + + Used instead of the full TrainingCheck to avoid requiring 6-timestamp + run leaves, results.json, dlio_config/, etc. for the integration tests. + The D-89 pool-walk logic in closed_submission_checksum is inherited + directly from TrainingCheck — this class just skips all other rules. + """ + + def __init__(self, log, config, submissions_logs): + from mlpstorage_py.submission_checker.loader import SubmissionLogs + super().__init__(log=log, path=submissions_logs.loader_metadata.folder) + self.config = config + self.submissions_logs = submissions_logs + self.mode = self.submissions_logs.loader_metadata.mode + self.model = self.submissions_logs.loader_metadata.benchmark + import os + self.run_path = os.path.join(self.path, "run") + self.name = "pool-aware training checks" + # Only register closed_submission_checksum (CHECK-05) + self.checks = [self.closed_submission_checksum] + + # Inherit closed_submission_checksum (CHECK-05) verbatim from TrainingCheck + closed_submission_checksum = TrainingCheck.closed_submission_checksum + + # --------------------------------------------------------------------------- # Shared run() invocation helper # --------------------------------------------------------------------------- @@ -220,13 +266,12 @@ def _run_args(root: Path, tmp_path: Path, version: str = "v3.0") -> argparse.Nam # =========================================================================== class TestPhase8SuccessCriteria: - """ROADMAP Phase 8 success criteria SC-1..SC-4 integration tests. + """ROADMAP Phase 8 success criteria SC-1..SC-5 integration tests. Each test method exercises run() against a v1.1 tree. Non-pool checks (SubmissionStructureCheck, SystemYamlSchemaCheck) are monkeypatched to - no-op so the test focuses on pool behavior. The training checker list - in MODE_TO_CHECKERS is emptied so the full TrainingCheck battery does - not run against the minimal v1.1 fixtures. + no-op so the test focuses on pool behavior. DirectoryCheck is excluded + from MODE_TO_CHECKERS via monkeypatch, replaced by _PoolAwareTrainingCheck. """ def _patch_non_pool_checks(self, monkeypatch): @@ -253,12 +298,12 @@ def _noop_ssc(self): ) def _patch_mode_checkers(self, monkeypatch): - """Monkeypatch MODE_TO_CHECKERS to an empty training checker list.""" + """Monkeypatch MODE_TO_CHECKERS to use _PoolAwareTrainingCheck only.""" import mlpstorage_py.submission_checker.main as main_mod monkeypatch.setattr( main_mod, "MODE_TO_CHECKERS", - {"training": []}, + {"training": [_PoolAwareTrainingCheck]}, ) # ------------------------------------------------------------------- @@ -283,7 +328,7 @@ def test_sc1_valid_v11_tree_passes( r for r in caplog.records if r.levelno >= logging.ERROR and any(kw in r.getMessage() for kw in [ - "CHECK-01", "CHECK-02", "CHECK-03", "CHECK-04", + "CHECK-01", "CHECK-02", "CHECK-03", "CHECK-04", "CHECK-05", "mlps-code-image", "pool image", "orphan", "Legacy code/", ]) ] @@ -450,3 +495,102 @@ def test_sc4_legacy_code_dir_returns_1( f"SC-4b: expected 'Legacy code/ layout detected', got: {all_messages}" ) + # ------------------------------------------------------------------- + # SC-5: two-version, two-pool-image tree → run() returns 0 (D-86/D-87) + # ------------------------------------------------------------------- + + def test_sc5_two_versions_two_images_passes( + self, tmp_path, v11_tree_factory, monkeypatch, caplog + ): + """SC-5: two run leaves each referencing a distinct pool image at different + mlpstorage_version values; REFERENCE_CHECKSUMS monkeypatched with correct + MD5 for each → run() returns 0. + + Exercises the D-86 per-image version-keyed lookup in + TrainingCheck.closed_submission_checksum (CHECK-05). + """ + self._patch_non_pool_checks(monkeypatch) + self._patch_mode_checkers(monkeypatch) + + log = _MockLog() + root = tmp_path / "submission_sc5" + root.mkdir(parents=True, exist_ok=True) + orgname = "Acme" + org_root = root / orgname + org_root.mkdir(parents=True, exist_ok=True) + + # Sentinel + (org_root / ".mlps-image-pool").write_text("2026-01-01T00:00:00Z\n") + + # Pool image 1: mlpstorage_version = "test-1.0" + content1 = "[project]\nname='image1'\nversion='1.0'\n" + pool_dir1, full_hash1 = _build_pool_image( + org_root, content=content1, mlpstorage_version="test-1.0", log=log + ) + + # Pool image 2: different content so different hash; version = "test-1.1" + content2 = "[project]\nname='image2'\nversion='2.0'\n" + pool_dir2, full_hash2 = _build_pool_image( + org_root, content=content2, mlpstorage_version="test-1.1", log=log + ) + + assert full_hash1 != full_hash2, "Pool images must have distinct hashes" + + # Compute correct MD5 for each pool image (what REFERENCE_CHECKSUMS should map to) + md5_for_image1 = compute_code_tree_md5(str(pool_dir1), log) + md5_for_image2 = compute_code_tree_md5(str(pool_dir2), log) + + # Two run leaf dirs: leaf1 → pool_dir1, leaf2 → pool_dir2 + leaf1 = ( + root / "closed" / orgname / "results" / "sys1" + / "training" / "unet3d" / "run" / "20260101_120000" + ) + leaf1.mkdir(parents=True, exist_ok=True) + (leaf1 / "output.txt").write_text("run 0\n") + _write_pointer_atomic(leaf1, full_hash1, log) + + leaf2 = ( + root / "closed" / orgname / "results" / "sys1" + / "training" / "unet3d" / "run" / "20260101_120001" + ) + leaf2.mkdir(parents=True, exist_ok=True) + (leaf2 / "output.txt").write_text("run 1\n") + _write_pointer_atomic(leaf2, full_hash2, log) + + # Systems + systems_dir = root / "closed" / orgname / "systems" + systems_dir.mkdir(parents=True, exist_ok=True) + (systems_dir / "sys1.yaml").write_text("system_under_test:\n deployment: onprem\n") + (systems_dir / "sys1.pdf").write_bytes(b"%PDF-1.0 minimal") + + # Monkeypatch REFERENCE_CHECKSUMS in both modules so D-86 lookup succeeds + patched_checksums = { + "test-1.0": md5_for_image1, + "test-1.1": md5_for_image2, + } + monkeypatch.setattr( + "mlpstorage_py.submission_checker.constants.REFERENCE_CHECKSUMS", + patched_checksums, + ) + monkeypatch.setattr( + "mlpstorage_py.submission_checker.checks.training_checks.REFERENCE_CHECKSUMS", + patched_checksums, + ) + + args = _run_args(root, tmp_path) + with caplog.at_level(logging.DEBUG): + rc = run(args) + + # Should pass: both pool images have correct REFERENCE_CHECKSUMS entries + errors = [ + r for r in caplog.records + if r.levelno >= logging.ERROR + and any(kw in r.getMessage() for kw in [ + "CHECK-01", "CHECK-02", "CHECK-03", "CHECK-04", "CHECK-05", + "mlps-code-image", "pool image", "orphan", "code-hash", + ]) + ] + assert rc == 0, ( + f"SC-5: expected exit code 0 for two-version two-image tree, got {rc}. " + f"Pool errors: {[r.getMessage() for r in errors]}" + )