diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ec432bd6a1..f9229a5841 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -42,6 +42,13 @@ # extension teardown. Bounding arenas fixes it without user-visible cost. os.environ.setdefault("MALLOC_ARENA_MAX", "2") +# MLflow 3.x put the filesystem (file://) tracking backend into "maintenance +# mode" and raises unless this opt-out is set. The MLflow exporter integration +# tests use file:// stores under tmp_path, so without this they fail with +# MlflowException and write zero runs. The exporter itself works fine against a +# file store once opted in. setdefault so an explicit override still wins. +os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true") + _logger = AIPerfLogger(__name__) diff --git a/tests/unit/common/config/test_loader.py b/tests/unit/common/config/test_loader.py new file mode 100644 index 0000000000..6ff110275e --- /dev/null +++ b/tests/unit/common/config/test_loader.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Supersession marker for the v1 config-loader tests. + +The v1 loader (``aiperf.common.config.loader``) exposed +``_load_config_file`` / ``load_service_config`` / ``load_user_config`` -- +extension-dispatched JSON/YAML readers that built ``ServiceConfig`` / +``UserConfig`` objects with an ``AIPERF_CONFIG_*_FILE`` env-var fallback. None of +that exists on v2: ``aiperf.common.config`` is not a package, there is no +``ServiceConfig``, and the schema-2.x loader is YAML-only. + +The v2 loader is ``aiperf.config.loader.core`` (``load_config`` / +``load_config_from_string`` / ``load_config_dict`` / ``validate_config_file``), +and every behavior the v1 suite asserted is already covered against the v2 API: + +| v1 test | v2 coverage | +| -------------------------------------------------- | ----------- | +| TestLoadConfigFile.test_json_file / yaml / yml | tests/unit/config/test_loader_edge_cases.py (load_config / load_config_from_string) +| TestLoadConfigFile.test_file_not_found_raises | test_loader_edge_cases.py (load_config missing-file -> ConfigurationError) +| TestLoadConfigFile.test_unsupported_extension | n/a -- v2 loader is YAML-only (no extension dispatch) +| TestLoadConfigFile.test_empty_yaml / empty_json | test_loader_edge_cases.py (empty / "null" -> error) + test_loader_adversarial.py +| TestLoadConfigFile.test_non_mapping_json/yaml | test_loader_edge_cases.py (list / scalar -> "must be a mapping") +| TestLoadConfigFile.test_case_insensitive_extension | n/a -- YAML-only +| TestLoadServiceConfig.* | n/a -- no ServiceConfig / no service-file loader on v2 +| TestLoadUserConfig.* | tests/unit/config/test_end_to_end_config_flow.py + test_loader_edge_cases.py + +The whole v1 file is therefore DROPPED (dup / no-v2-home). This module keeps a +single guard so the supersession decision is self-verifying: if anyone re-adds +the v1 loader API, this test fails and forces a real port of the suite above. +""" + +from __future__ import annotations + +import importlib + + +def test_v1_config_loader_api_is_gone() -> None: + """The v1 ``aiperf.common.config.loader`` module must not exist on v2. + + Guards the supersession documented in this module's docstring: the v1 + loader entry points were replaced wholesale by ``aiperf.config.loader`` + (covered by tests/unit/config/test_loader_*.py). If this import ever starts + succeeding again, the v1 loader was re-introduced and its tests must be + properly re-ported rather than left as this stub. + """ + try: + importlib.import_module("aiperf.common.config.loader") + except ModuleNotFoundError: + return + raise AssertionError( + "aiperf.common.config.loader was re-introduced -- re-port the v1 " + "loader test suite (see tests/unit/config/test_loader_edge_cases.py / " + "test_loader_adversarial.py for the current v2 coverage)." + ) diff --git a/tests/unit/common/test_dataset_models_prereq.py b/tests/unit/common/test_dataset_models_prereq.py new file mode 100644 index 0000000000..da32eec03b --- /dev/null +++ b/tests/unit/common/test_dataset_models_prereq.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from aiperf.common.enums import PrerequisiteKind +from aiperf.common.models import Turn, TurnMetadata, TurnPrerequisite + + +def test_turn_defaults_empty_prerequisites(): + t = Turn() + assert t.prerequisites == [] + + +def test_turn_carries_prerequisites(): + p = TurnPrerequisite(kind=PrerequisiteKind.SPAWN_JOIN, branch_id="b1") + t = Turn(prerequisites=[p]) + assert len(t.prerequisites) == 1 + assert t.prerequisites[0].branch_id == "b1" + + +def test_turn_metadata_carries_prerequisites(): + p = TurnPrerequisite(kind=PrerequisiteKind.SPAWN_JOIN, branch_id="b1") + m = TurnMetadata(prerequisites=[p]) + assert m.prerequisites == [p] + + +def test_turn_metadata_copied_from_turn(): + p = TurnPrerequisite(kind=PrerequisiteKind.SPAWN_JOIN, branch_id="b1") + t = Turn(prerequisites=[p], branch_ids=["b1"]) + m = t.metadata() + assert m.prerequisites == [p] + assert m.branch_ids == ["b1"] diff --git a/tests/unit/orchestrator/test_orchestrator_execution.py b/tests/unit/orchestrator/test_orchestrator_execution.py new file mode 100644 index 0000000000..f0a65506ec --- /dev/null +++ b/tests/unit/orchestrator/test_orchestrator_execution.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Supersession marker for the v1 orchestrator execution-method tests. + +The v1 suite exercised the agentx orchestrator's strategy auto-detection and +generic execution loop, all driven off a v1 ``UserConfig``: + +- ``_resolve_strategy(config)`` mapping ``loadgen.concurrency`` (scalar vs list) + + ``loadgen.num_profile_runs`` to a ``ParameterSweepStrategy`` / + ``FixedTrialsStrategy`` / ``SweepConfidenceStrategy`` (and raising for a + single run); +- ``_execute(config, strategy)`` choosing the strategy's own ``execute()`` + (Option B) or falling back to ``_execute_loop``; +- ``_execute_loop(config, strategy)`` with ``tag_result`` / ``get_run_label`` / + ``get_run_path`` / ``collect_failed_values`` / cooldown via ``time.sleep`` and + "sweep values failed" warnings; +- ``SweepConfidenceStrategy.execute()`` iteration order + metadata tagging + + cooldowns for repeated vs independent modes; +- the ``_create_sweep_strategy`` / ``_create_confidence_strategy`` factories. + +main's #1035 removed ALL of these from the orchestrator. The v2 +``MultiRunOrchestrator`` takes a pre-built ``BenchmarkPlan`` (the CLI runner +already resolved variations + strategy at plan-build time) and a ``RunExecutor``; +it has no ``_resolve_strategy`` / ``_execute`` / ``_execute_loop`` / +``_create_*_strategy``, no ``UserConfig``, and no in-orchestrator ``time.sleep`` +(cooldown reads ``plan.sweep.cooldown_seconds``). The iteration order / +metadata-tagging / cooldown concerns these tests covered are now expressed +through ``BenchmarkPlan`` + ``SweepMode`` and verified on v2: + +| v1 test concern | v2 coverage | +| ----------------------------------------------------- | ----------- | +| _resolve_strategy auto-detect from config | tests/unit/orchestrator/test_strategies.py + cli_runner._strategy.build_strategy | +| _execute Option-B vs generic loop | tests/unit/orchestrator/test_multi_run_orchestrator.py (execute -> _execute_repeated/_independent) | +| _execute_loop tag_result / failed-value warnings | test_multi_run_orchestrator.py (_stamp_variation_metadata, failure threshold) | +| repeated vs independent iteration order | test_multi_run_orchestrator.py (SweepMode REPEATED/INDEPENDENT order asserts) | +| repeated/independent cooldown application | test_multi_run_orchestrator.py (plan.sweep cooldown) | +| _create_sweep / _create_confidence factories | cli_runner._strategy.build_strategy + test_strategies.py | + +This module keeps a single guard so the supersession is self-verifying: if any +of the v1 execution methods re-appear on the orchestrator, the guard fails and +forces a real re-port of the suite mapped above. +""" + +from __future__ import annotations + +from aiperf.orchestrator.orchestrator import MultiRunOrchestrator + + +def test_v1_orchestrator_execution_methods_are_gone() -> None: + """The v1 strategy-resolution + generic-loop methods must not exist on v2. + + Guards the supersession documented in this module's docstring. On v2 the + orchestrator consumes a pre-resolved ``BenchmarkPlan`` + ``RunExecutor``; + strategy resolution moved to ``aiperf.cli_runner._strategy.build_strategy``. + If any v1 method re-appears, the v1 execution model was re-introduced and its + tests must be properly re-ported (see the table in this module's docstring) + rather than left as this stub. + """ + for v1_method in ( + "_resolve_strategy", + "_execute", + "_execute_loop", + "_create_sweep_strategy", + "_create_confidence_strategy", + ): + assert not hasattr(MultiRunOrchestrator, v1_method), ( + f"v1 orchestrator execution method {v1_method!r} re-appeared on v2 " + "MultiRunOrchestrator -- re-port the v1 suite (see module docstring)." + ) + + # The v2 plan-driven entry point must exist (catches a refactor that drops + # the real execute path while leaving this stub stale). + assert hasattr(MultiRunOrchestrator, "execute") + + # Strategy resolution re-homed onto the cli_runner strategy builder. + from aiperf.cli_runner import _strategy + + assert hasattr(_strategy, "build_strategy")