test: add v2 regression guards and supersession markers

tests/integration/conftest.py

@@ -42,6 +42,13 @@
 # extension teardown. Bounding arenas fixes it without user-visible cost.
 os.environ.setdefault("MALLOC_ARENA_MAX", "2")
 
+# MLflow 3.x put the filesystem (file://) tracking backend into "maintenance
+# mode" and raises unless this opt-out is set. The MLflow exporter integration
+# tests use file:// stores under tmp_path, so without this they fail with
+# MlflowException and write zero runs. The exporter itself works fine against a
+# file store once opted in. setdefault so an explicit override still wins.
+os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true")
+
 _logger = AIPerfLogger(__name__)
 
 

tests/unit/common/config/test_loader.py

@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Supersession marker for the v1 config-loader tests.
+
+The v1 loader (``aiperf.common.config.loader``) exposed
+``_load_config_file`` / ``load_service_config`` / ``load_user_config`` --
+extension-dispatched JSON/YAML readers that built ``ServiceConfig`` /
+``UserConfig`` objects with an ``AIPERF_CONFIG_*_FILE`` env-var fallback. None of
+that exists on v2: ``aiperf.common.config`` is not a package, there is no
+``ServiceConfig``, and the schema-2.x loader is YAML-only.
+
+The v2 loader is ``aiperf.config.loader.core`` (``load_config`` /
+``load_config_from_string`` / ``load_config_dict`` / ``validate_config_file``),
+and every behavior the v1 suite asserted is already covered against the v2 API:
+
+| v1 test                                            | v2 coverage |
+| -------------------------------------------------- | ----------- |
+| TestLoadConfigFile.test_json_file / yaml / yml     | tests/unit/config/test_loader_edge_cases.py (load_config / load_config_from_string)
+| TestLoadConfigFile.test_file_not_found_raises      | test_loader_edge_cases.py (load_config missing-file -> ConfigurationError)
+| TestLoadConfigFile.test_unsupported_extension      | n/a -- v2 loader is YAML-only (no extension dispatch)
+| TestLoadConfigFile.test_empty_yaml / empty_json    | test_loader_edge_cases.py (empty / "null" -> error) + test_loader_adversarial.py
+| TestLoadConfigFile.test_non_mapping_json/yaml      | test_loader_edge_cases.py (list / scalar -> "must be a mapping")
+| TestLoadConfigFile.test_case_insensitive_extension | n/a -- YAML-only
+| TestLoadServiceConfig.*                            | n/a -- no ServiceConfig / no service-file loader on v2
+| TestLoadUserConfig.*                               | tests/unit/config/test_end_to_end_config_flow.py + test_loader_edge_cases.py
+
+The whole v1 file is therefore DROPPED (dup / no-v2-home). This module keeps a
+single guard so the supersession decision is self-verifying: if anyone re-adds
+the v1 loader API, this test fails and forces a real port of the suite above.
+"""
+
+from __future__ import annotations
+
+import importlib
+
+
+def test_v1_config_loader_api_is_gone() -> None:
+    """The v1 ``aiperf.common.config.loader`` module must not exist on v2.
+
+    Guards the supersession documented in this module's docstring: the v1
+    loader entry points were replaced wholesale by ``aiperf.config.loader``
+    (covered by tests/unit/config/test_loader_*.py). If this import ever starts
+    succeeding again, the v1 loader was re-introduced and its tests must be
+    properly re-ported rather than left as this stub.
+    """
+    try:
+        importlib.import_module("aiperf.common.config.loader")
+    except ModuleNotFoundError:
+        return
+    raise AssertionError(
+        "aiperf.common.config.loader was re-introduced -- re-port the v1 "
+        "loader test suite (see tests/unit/config/test_loader_edge_cases.py / "
+        "test_loader_adversarial.py for the current v2 coverage)."
+    )

tests/unit/common/test_dataset_models_prereq.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+from aiperf.common.enums import PrerequisiteKind
+from aiperf.common.models import Turn, TurnMetadata, TurnPrerequisite
+
+
+def test_turn_defaults_empty_prerequisites():
+    t = Turn()
+    assert t.prerequisites == []
+
+
+def test_turn_carries_prerequisites():
+    p = TurnPrerequisite(kind=PrerequisiteKind.SPAWN_JOIN, branch_id="b1")
+    t = Turn(prerequisites=[p])
+    assert len(t.prerequisites) == 1
+    assert t.prerequisites[0].branch_id == "b1"
+
+
+def test_turn_metadata_carries_prerequisites():
+    p = TurnPrerequisite(kind=PrerequisiteKind.SPAWN_JOIN, branch_id="b1")
+    m = TurnMetadata(prerequisites=[p])
+    assert m.prerequisites == [p]
+
+
+def test_turn_metadata_copied_from_turn():
+    p = TurnPrerequisite(kind=PrerequisiteKind.SPAWN_JOIN, branch_id="b1")
+    t = Turn(prerequisites=[p], branch_ids=["b1"])
+    m = t.metadata()
+    assert m.prerequisites == [p]
+    assert m.branch_ids == ["b1"]

tests/unit/orchestrator/test_orchestrator_execution.py

@@ -0,0 +1,78 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Supersession marker for the v1 orchestrator execution-method tests.
+
+The v1 suite exercised the agentx orchestrator's strategy auto-detection and
+generic execution loop, all driven off a v1 ``UserConfig``:
+
+- ``_resolve_strategy(config)`` mapping ``loadgen.concurrency`` (scalar vs list)
+  + ``loadgen.num_profile_runs`` to a ``ParameterSweepStrategy`` /
+  ``FixedTrialsStrategy`` / ``SweepConfidenceStrategy`` (and raising for a
+  single run);
+- ``_execute(config, strategy)`` choosing the strategy's own ``execute()``
+  (Option B) or falling back to ``_execute_loop``;
+- ``_execute_loop(config, strategy)`` with ``tag_result`` / ``get_run_label`` /
+  ``get_run_path`` / ``collect_failed_values`` / cooldown via ``time.sleep`` and
+  "sweep values failed" warnings;
+- ``SweepConfidenceStrategy.execute()`` iteration order + metadata tagging +
+  cooldowns for repeated vs independent modes;
+- the ``_create_sweep_strategy`` / ``_create_confidence_strategy`` factories.
+
+main's #1035 removed ALL of these from the orchestrator. The v2
+``MultiRunOrchestrator`` takes a pre-built ``BenchmarkPlan`` (the CLI runner
+already resolved variations + strategy at plan-build time) and a ``RunExecutor``;
+it has no ``_resolve_strategy`` / ``_execute`` / ``_execute_loop`` /
+``_create_*_strategy``, no ``UserConfig``, and no in-orchestrator ``time.sleep``
+(cooldown reads ``plan.sweep.cooldown_seconds``). The iteration order /
+metadata-tagging / cooldown concerns these tests covered are now expressed
+through ``BenchmarkPlan`` + ``SweepMode`` and verified on v2:
+
+| v1 test concern                                       | v2 coverage |
+| ----------------------------------------------------- | ----------- |
+| _resolve_strategy auto-detect from config             | tests/unit/orchestrator/test_strategies.py + cli_runner._strategy.build_strategy |
+| _execute Option-B vs generic loop                     | tests/unit/orchestrator/test_multi_run_orchestrator.py (execute -> _execute_repeated/_independent) |
+| _execute_loop tag_result / failed-value warnings      | test_multi_run_orchestrator.py (_stamp_variation_metadata, failure threshold) |
+| repeated vs independent iteration order               | test_multi_run_orchestrator.py (SweepMode REPEATED/INDEPENDENT order asserts) |
+| repeated/independent cooldown application             | test_multi_run_orchestrator.py (plan.sweep cooldown) |
+| _create_sweep / _create_confidence factories          | cli_runner._strategy.build_strategy + test_strategies.py |
+
+This module keeps a single guard so the supersession is self-verifying: if any
+of the v1 execution methods re-appear on the orchestrator, the guard fails and
+forces a real re-port of the suite mapped above.
+"""
+
+from __future__ import annotations
+
+from aiperf.orchestrator.orchestrator import MultiRunOrchestrator
+
+
+def test_v1_orchestrator_execution_methods_are_gone() -> None:
+    """The v1 strategy-resolution + generic-loop methods must not exist on v2.
+
+    Guards the supersession documented in this module's docstring. On v2 the
+    orchestrator consumes a pre-resolved ``BenchmarkPlan`` + ``RunExecutor``;
+    strategy resolution moved to ``aiperf.cli_runner._strategy.build_strategy``.
+    If any v1 method re-appears, the v1 execution model was re-introduced and its
+    tests must be properly re-ported (see the table in this module's docstring)
+    rather than left as this stub.
+    """
+    for v1_method in (
+        "_resolve_strategy",
+        "_execute",
+        "_execute_loop",
+        "_create_sweep_strategy",
+        "_create_confidence_strategy",
+    ):
+        assert not hasattr(MultiRunOrchestrator, v1_method), (
+            f"v1 orchestrator execution method {v1_method!r} re-appeared on v2 "
+            "MultiRunOrchestrator -- re-port the v1 suite (see module docstring)."
+        )
+
+    # The v2 plan-driven entry point must exist (catches a refactor that drops
+    # the real execute path while leaving this stub stale).
+    assert hasattr(MultiRunOrchestrator, "execute")
+
+    # Strategy resolution re-homed onto the cli_runner strategy builder.
+    from aiperf.cli_runner import _strategy
+
+    assert hasattr(_strategy, "build_strategy")