diff --git a/conftest.py b/conftest.py index 7bd50d75a..e31510cdb 100644 --- a/conftest.py +++ b/conftest.py @@ -429,6 +429,7 @@ def mutable_config(tmpdir_factory, configuration_dir): ramble.config.ConfigScope(name, str(mutable_dir.join(name))) for name in ["site", "system", "user"] ] + scopes.append(ramble.config.InternalConfigScope("command_line")) with ramble.config.use_configuration(*scopes) as cfg: yield cfg @@ -442,6 +443,7 @@ def mutable_empty_config(tmpdir_factory, configuration_dir): ramble.config.ConfigScope(name, str(mutable_dir.join(name))) for name in ["site", "system", "user"] ] + scopes.append(ramble.config.InternalConfigScope("command_line")) with ramble.config.use_configuration(*scopes) as cfg: yield cfg diff --git a/lib/ramble/ramble/cmd/help.py b/lib/ramble/ramble/cmd/help.py index 965053482..6c6a7ab30 100644 --- a/lib/ramble/ramble/cmd/help.py +++ b/lib/ramble/ramble/cmd/help.py @@ -77,3 +77,4 @@ def help(parser, args): parser.parse_args([args.help_command, "-h"]) else: sys.stdout.write(parser.format_help(level=args.all)) + return 0 diff --git a/lib/ramble/ramble/cmd/workspace.py b/lib/ramble/ramble/cmd/workspace.py index e4fd3c995..a722170d2 100644 --- a/lib/ramble/ramble/cmd/workspace.py +++ b/lib/ramble/ramble/cmd/workspace.py @@ -704,7 +704,6 @@ def workspace_push_to_cache(args): pipeline = pipeline_cls(ws, filters, spack_cache_path=args.cache_path) workspace_run_pipeline(args, pipeline) - pipeline.run() def workspace_push_to_cache_setup_parser(subparser): diff --git a/lib/ramble/ramble/main.py b/lib/ramble/ramble/main.py index 131389803..e0a0673ed 100644 --- a/lib/ramble/ramble/main.py +++ b/lib/ramble/ramble/main.py @@ -332,6 +332,8 @@ def make_argument_parser(**kwargs): # See https://docs.python.org/3/library/argparse.html#color. kwargs.pop("color") + kwargs.setdefault("prog", "ramble") + parser = RambleArgumentParser( formatter_class=RambleHelpFormatter, add_help=False, diff --git a/lib/ramble/ramble/test/cmd/clean.py b/lib/ramble/ramble/test/cmd/clean.py index da561e49e..0f12ffe65 100644 --- a/lib/ramble/ramble/test/cmd/clean.py +++ b/lib/ramble/ramble/test/cmd/clean.py @@ -34,29 +34,41 @@ def __call__(self, *args, **kwargs): monkeypatch.setattr(ramble.caches.fetch_cache, "destroy", Counter("downloads"), raising=False) monkeypatch.setattr(ramble.caches.misc_cache, "destroy", Counter("caches")) monkeypatch.setattr(ramble.cmd.clean, "remove_python_caches", Counter("python_caches")) + monkeypatch.setattr(ramble.cmd.clean, "remove_reports_files", Counter("reports")) yield counts -all_effects = ["downloads", "caches", "python_caches"] +# All possible effect categories monitored by the test harness +all_monitored_effects = ["downloads", "caches", "python_caches", "reports"] + +# '-a' / '--all' is defined as AllClean (equivalent to -dmp) and excludes 'reports' +all_flag_effects = ["downloads", "caches", "python_caches"] @pytest.mark.usefixtures("config") @pytest.mark.parametrize( - "command_line,effects", + "args,effects", [ - ("-d", ["downloads"]), - ("-m", ["caches"]), - ("-p", ["python_caches"]), - ("-a", all_effects), + ([], ["downloads"]), + (["-d"], ["downloads"]), + (["--downloads"], ["downloads"]), + (["-m"], ["caches"]), + (["--misc-cache"], ["caches"]), + (["-p"], ["python_caches"]), + (["--python-cache"], ["python_caches"]), + (["-r"], ["reports"]), + (["--reports"], ["reports"]), + (["-a"], all_flag_effects), + (["--all"], all_flag_effects), ], ) -def test_function_calls(command_line, effects, mock_calls_for_clean): +def test_function_calls(args, effects, mock_calls_for_clean): # Call the command with the supplied command line - clean(command_line) + clean(*args) # Assert that we called the expected functions the correct # number of times - for name in all_effects: + for name in all_monitored_effects: assert mock_calls_for_clean[name] == (1 if name in effects else 0) diff --git a/lib/ramble/ramble/test/cmd/commands.py b/lib/ramble/ramble/test/cmd/commands.py new file mode 100644 index 000000000..8c1c8a9ac --- /dev/null +++ b/lib/ramble/ramble/test/cmd/commands.py @@ -0,0 +1,87 @@ +# Copyright 2022-2026 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +from unittest.mock import patch + +import pytest + +import ramble.cmd +import ramble.config +from ramble.error import RambleCommandError +from ramble.main import RambleCommand +from ramble.util.logger import logger + +command = RambleCommand("commands") + + +def test_missing_command(): + with pytest.raises(RambleCommandError) as err_info: + RambleCommand("missing-command") + + assert "does not exist" in str(err_info.value) + + +def test_available_command(): + for cmd in ramble.cmd.all_commands(): + logger.msg(f"Command = {cmd}") + RambleCommand(cmd) + + +def test_command_output(tmpdir): + formats = ["subcommands", "rst", "names", "bash"] + for f in formats: + file = os.path.join(tmpdir, f"outfile.{f}") + command("--format", f, "--update", file) + assert os.path.isfile(file) + + target = os.path.join(tmpdir, "outfile.names") + header = os.path.join(tmpdir, "outfile.subcommands") + command("--update", target, "--header", header, "-a") + assert os.path.isfile(target) + + +def test_command_alias_output(mutable_config): + with ramble.config.override("config:aliases", {"ws": "workspace"}): + out = command("-a", output=str) + assert "ws" in out + assert "workspace" in out + + +def test_command_invalid_header(tmpdir): + missing_header = os.path.join(tmpdir, "nonexistent_header.txt") + out = command("--header", missing_header, fail_on_error=False) + assert "No such file" in out + + +def test_command_update_completion_conflict(): + out = command("--update-completion", "-a", fail_on_error=False) + assert "--update-completion can only be specified alone" in out + + +def test_command_update_completion(tmpdir): + bash_no_aliases = str(tmpdir.join("ramble-completion.bash")) + base_with_aliases = str(tmpdir.join("custom-ramble-completion.bash")) + mock_args = { + "bash_no_aliases": { + "aliases": False, + "format": "bash", + "header": os.path.join(ramble.paths.share_path, "bash", "ramble-completion.in"), + "update": bash_no_aliases, + }, + "base_with_aliases": { + "aliases": True, + "format": "bash", + "header": os.path.join(ramble.paths.share_path, "bash", "ramble-completion.in"), + "update": base_with_aliases, + }, + } + with patch.dict("ramble.cmd.commands.update_completion_args", mock_args): + command("--update-completion") + assert os.path.isfile(bash_no_aliases) + assert os.path.isfile(base_with_aliases) diff --git a/lib/ramble/ramble/test/cmd/config.py b/lib/ramble/ramble/test/cmd/config.py index 4f7ef9451..2485c7738 100644 --- a/lib/ramble/ramble/test/cmd/config.py +++ b/lib/ramble/ramble/test/cmd/config.py @@ -831,3 +831,18 @@ def test_command_alias(mutable_config): output = f.getvalue() assert ret == 0 assert "ramble info" not in output + + +def test_config_blame(mock_low_high_config): + low_path = mock_low_high_config.scopes["low"].path + fs.mkdirp(low_path) + with open(os.path.join(low_path, "config.yaml"), "w", encoding="utf-8") as f: + f.write("""\ +config: + verbose: true +""") + + output = config("blame", "config") + assert "config:" in output + assert "verbose: true" in output + assert "config.yaml:" in output diff --git a/lib/ramble/ramble/test/cmd/help.py b/lib/ramble/ramble/test/cmd/help.py index 85d6cc66f..10efe9496 100644 --- a/lib/ramble/ramble/test/cmd/help.py +++ b/lib/ramble/ramble/test/cmd/help.py @@ -6,6 +6,8 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import pytest + from ramble.main import RambleCommand help_cmd = RambleCommand("help") @@ -13,5 +15,35 @@ def test_help_default(): """Test that `ramble help` gives short help.""" + help_cmd = RambleCommand("help") output = help_cmd() assert "A flexible benchmark experiment manager" in output + + +def test_help_all(): + """Test that `ramble help --all` and `-a` list all available commands.""" + help_cmd = RambleCommand("help") + out_all = help_cmd("--all") + assert "Complete list of ramble commands:" in out_all + assert "workspace" in out_all + + help_cmd2 = RambleCommand("help") + out_short = help_cmd2("-a") + assert "Complete list of ramble commands:" in out_short + assert "workspace" in out_short + + +def test_help_spec(): + """Test that `ramble help --spec` prints the spec guide.""" + help_cmd = RambleCommand("help") + output = help_cmd("--spec") + assert "spec expression syntax:" in output + assert "application [constraint]" in output + + +@pytest.mark.parametrize("subcmd", ["config", "info", "list", "workspace", "help"]) +def test_help_command(subcmd): + """Test that `ramble help ` prints help for the given command.""" + help_cmd = RambleCommand("help") + output = help_cmd(subcmd) + assert f"usage: ramble {subcmd}" in output or "usage: ramble" in output diff --git a/lib/ramble/ramble/test/cmd/main.py b/lib/ramble/ramble/test/cmd/main.py index 781c2a573..fd1e217b0 100644 --- a/lib/ramble/ramble/test/cmd/main.py +++ b/lib/ramble/ramble/test/cmd/main.py @@ -27,3 +27,41 @@ def test_command_argument_before_command_is_rejected(capsys): captured = capsys.readouterr() assert ret == 1 assert "unrecognized arguments: --dry-run" in captured.err + + +@pytest.mark.parametrize("flag", ["-V", "--version"]) +def test_global_version(flag, capsys): + ret = ramble.main.main(argv=[flag]) + captured = capsys.readouterr() + assert ret == 0 + assert str(ramble.ramble_version) in captured.out + + +@pytest.mark.parametrize("flag", ["-h", "--help"]) +def test_global_help(flag, capsys): + ret = ramble.main.main(argv=[flag]) + captured = capsys.readouterr() + assert ret == 0 + assert "A flexible benchmark experiment manager" in captured.out + + +@pytest.mark.parametrize("flag", ["-H", "--all-help"]) +def test_global_all_help(flag, capsys): + ret = ramble.main.main(argv=[flag]) + captured = capsys.readouterr() + assert ret == 0 + assert "Complete list of ramble commands:" in captured.out + + +def test_global_config_var(capsys): + ret = ramble.main.main(argv=["-c", "config:debug:true", "help"]) + captured = capsys.readouterr() + assert ret == 0 + assert "A flexible benchmark experiment manager" in captured.out + + +def test_global_config_scope(tmpdir, capsys): + ret = ramble.main.main(argv=["-C", str(tmpdir), "help"]) + captured = capsys.readouterr() + assert ret == 0 + assert "A flexible benchmark experiment manager" in captured.out diff --git a/lib/ramble/ramble/test/cmd/python.py b/lib/ramble/ramble/test/cmd/python.py index 42aa72ba2..28df16873 100644 --- a/lib/ramble/ramble/test/cmd/python.py +++ b/lib/ramble/ramble/test/cmd/python.py @@ -45,3 +45,23 @@ def test_python_with_module(): def test_python_raises(): out = python("--foobar", fail_on_error=False) assert "Error: Unknown arguments" in out + + +def test_python_script(tmpdir): + script = tmpdir.join("test_script.py") + script.write(""" +import sys +import ramble +print(f"ARG:{sys.argv[1]}") +print(f"VER:{ramble.ramble_version}") +""") + out = python(str(script), "hello") + assert "ARG:hello" in out + assert f"VER:{ramble.ramble_version}" in out + + +def test_python_command_and_script(tmpdir): + script = tmpdir.join("dummy.py") + script.write("print('hello')") + out = python("-c", "print('cmd')", str(script), fail_on_error=False) + assert "You can only specify a command OR script" in out diff --git a/lib/ramble/ramble/test/cmd/results.py b/lib/ramble/ramble/test/cmd/results.py index 81ae6852e..0fadb1745 100644 --- a/lib/ramble/ramble/test/cmd/results.py +++ b/lib/ramble/ramble/test/cmd/results.py @@ -6,15 +6,60 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import json import os +from unittest.mock import patch import pytest import ramble.cmd.results import ramble.paths +from ramble.main import RambleCommand INPUT_DATA = os.path.join(ramble.paths.test_path, "data", "results_upload") +results = RambleCommand("results") + + +@pytest.fixture +def sample_results_file(tmpdir): + data = { + "workspace_name": "test_ws", + "experiments": [ + { + "name": "hostname.local.test_exp", + "RAMBLE_STATUS": "SUCCESS", + "experiment_name": "test_exp", + "experiment_namespace": "hostname.local.test_exp", + "application_name": "hostname", + "workload_name": "local", + "workload_namespace": "hostname.local", + "context_name": "null", + "RAMBLE_VARIABLES": {"n_nodes": "1", "n_ranks": "1", "repeat_index": "0"}, + "RAMBLE_RAW_VARIABLES": {"n_nodes": "1", "n_ranks": "1", "repeat_index": "0"}, + "CONTEXTS": [ + { + "name": "null", + "display_name": "null", + "foms": [ + { + "name": "runtime", + "value": 1.23, + "units": "s", + "origin": "hostname", + "origin_type": "application", + } + ], + } + ], + } + ], + } + file_path = tmpdir.join("results.latest.json") + with open(file_path, "w", encoding="utf-8") as f: + json.dump(data, f) + return str(file_path) + @pytest.mark.parametrize( "filename,expected_output", @@ -38,3 +83,38 @@ def test_file_import_rejects_invalid_files(filename, expected_output, capsys): ramble.cmd.results.import_results_file(filename) captured = capsys.readouterr().err assert expected_output in captured + + +def test_results_upload(sample_results_file): + with patch("ramble.uploader.upload_results") as mock_upload: + results("upload", sample_results_file) + mock_upload.assert_called_once() + args, _ = mock_upload.call_args + assert args[0]["workspace_name"] == "test_ws" + assert len(args[0]["experiments"]) == 1 + + +def test_results_index(sample_results_file): + out = results("index", "-f", sample_results_file) + assert "FOMs:" in out + assert "runtime" in out + + out_v = results("index", "-v", "-f", sample_results_file) + assert "All Variables" in out_v + assert "n_nodes" in out_v + + +def test_results_report(sample_results_file): + with patch("ramble.reports.make_report") as mock_make_report: + results("report", "--foms", "-f", sample_results_file) + mock_make_report.assert_called_once() + + +def test_results_missing_file(): + out = results("index", "-f", "nonexistent_file.json", fail_on_error=False) + assert "Cannot find file" in out + + +def test_results_no_workspace_no_file(): + out = results("index", fail_on_error=False) + assert "requires either a results filename" in out diff --git a/lib/ramble/ramble/test/cmd/workspace.py b/lib/ramble/ramble/test/cmd/workspace.py index 63366d293..1a91ec9f5 100644 --- a/lib/ramble/ramble/test/cmd/workspace.py +++ b/lib/ramble/ramble/test/cmd/workspace.py @@ -3405,3 +3405,103 @@ def test_workspace_experiment_logs(workspace_name): ["experiments", "basic", "test_wl", "generated", "generated.out"] ) assert expected_output in output + + +def test_workspace_ls(workspace_name): + with ramble.workspace.create(workspace_name) as ws: + ws.write() + out = workspace("ls") + assert workspace_name in out + + +def test_workspace_rm_alias(workspace_name): + workspace("create", workspace_name) + out = workspace("list") + assert workspace_name in out + + workspace("rm", "-y", workspace_name) + out = workspace("list") + assert workspace_name not in out + + +def test_workspace_push_to_cache(workspace_name, tmpdir): + global_args = ["-w", workspace_name] + cache_dir = tmpdir.mkdir("buildcache") + + with ramble.workspace.create(workspace_name) as ws: + ws.write() + workspace( + "manage", + "experiments", + "basic", + "--wf", + "test_wl", + "-v", + "n_ranks=1", + "-v", + "n_nodes=1", + "--default-variable-value", + "1", + global_args=global_args, + ) + workspace("concretize", global_args=global_args) + workspace( + "push-to-cache", + "-d", + str(cache_dir), + "--dry-run", + global_args=global_args, + ) + + +def test_workspace_bootstrap_command(workspace_name): + global_args = ["-w", workspace_name] + + with ramble.workspace.create(workspace_name) as ws: + ws.write() + workspace( + "manage", + "experiments", + "basic", + "--wf", + "test_wl", + "-v", + "n_ranks=1", + "-v", + "n_nodes=1", + "--default-variable-value", + "1", + global_args=global_args, + ) + workspace("concretize", global_args=global_args) + workspace("bootstrap", global_args=global_args) + + +def test_workspace_manage_filter_groups_rm_alias(workspace_name): + global_args = ["-w", workspace_name] + + with ramble.workspace.create(workspace_name) as ws: + ws.write() + workspace( + "manage", + "filter-groups", + "add", + "-n", + "test-group", + "--where", + "{n_nodes} == 1", + global_args=global_args, + ) + out = workspace("manage", "filter-groups", "list", global_args=global_args) + assert "test-group" in out + + workspace( + "manage", + "filter-groups", + "rm", + "-n", + "test-group", + global_args=global_args, + ) + out = workspace("manage", "filter-groups", "list", global_args=global_args) + assert "test-group" not in out diff --git a/lib/ramble/ramble/test/commands.py b/lib/ramble/ramble/test/commands.py deleted file mode 100644 index cbf789b00..000000000 --- a/lib/ramble/ramble/test/commands.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2022-2026 The Ramble Authors -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -import os - -import pytest - -import ramble.config -from ramble.error import RambleCommandError -from ramble.main import RambleCommand -from ramble.util.logger import logger - -command = RambleCommand("commands") - - -def test_missing_command(): - with pytest.raises(RambleCommandError) as err_info: - RambleCommand("missing-command") - - assert "does not exist" in str(err_info.value) - - -def test_available_command(): - import ramble.cmd - - for command in ramble.cmd.all_commands(): - logger.msg(f"Command = {command}") - - RambleCommand(command) - - -def test_command_output(tmpdir): - formats = ["subcommands", "rst", "names", "bash"] - for f in formats: - file = os.path.join(tmpdir, f"outfile.{f}") - command("--format", f, "--update", file) - assert os.path.isfile(file) - - target = os.path.join(tmpdir, "outfile.names") - header = os.path.join(tmpdir, "outfile.subcommands") - command("--update", target, "--header", header, "-a") - - -def test_command_alias_output(mutable_config): - with ramble.config.override("config:aliases", {"ws": "workspace"}): - out = command("-a", output=str) - assert "ws" in out - assert "workspace" in out