Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cli/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ aws-cdk.aws-codebuild~=1.137
aws-cdk.aws-dynamodb~=1.137
aws-cdk.aws-ec2~=1.137
aws-cdk.aws-efs~=1.137
aws-cdk.aws-events~=1.137
aws-cdk.aws-events~=1.191
aws-cdk.aws-fsx~=1.137
aws-cdk.aws-imagebuilder~=1.137
aws-cdk.aws-iam~=1.137
Expand Down
23 changes: 21 additions & 2 deletions tests/integration-tests/tests/efa/test_efa.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@

import pytest
from assertpy import assert_that
from remote_command_executor import RemoteCommandExecutor
from utils import get_compute_nodes_instance_ids
from remote_command_executor import RemoteCommandExecutor, RemoteCommandExecutionError
from utils import get_compute_nodes_instance_ids, get_instance_info

from tests.common.assertions import assert_no_errors_in_logs
from tests.common.mpi_common import _test_mpi
Expand Down Expand Up @@ -66,6 +66,8 @@ def test_efa(

run_system_analyzer(cluster, scheduler_commands_factory, request, partition="efa-enabled")

_test_fsx_performance_tuning_for_large_instances(instance, remote_command_executor)

if instance in osu_benchmarks_instances:
benchmark_failures = []

Expand Down Expand Up @@ -138,6 +140,23 @@ def _test_efa_installation(scheduler_commands, remote_command_executor, efa_inst
assert_that(result.stdout).does_not_contain("1d0f:efa")


def _test_fsx_performance_tuning_for_large_instances(instance_type, remote_command_executor):
instance_info = get_instance_info(instance_type)
vcpu = instance_info.get("VCpuInfo").get("DefaultVCpus")
memory = instance_info.get("MemoryInfo").get("SizeInMiB") # 256 GiB * 1024 = 262144 MiB
if vcpu > 64:
remote_command_executor.run_remote_command("lctl get_param osc.*OST*.max_rpcs_in_flight | grep 32$")
remote_command_executor.run_remote_command("lctl get_param mdc.*.max_rpcs_in_flight | grep 64$")
remote_command_executor.run_remote_command("lctl get_param mdc.*.max_mod_rpcs_in_flight | grep 50$")
else:
with pytest.raises(RemoteCommandExecutionError):
remote_command_executor.run_remote_command("lctl get_param osc.*OST*.max_rpcs_in_flight | grep 32$")
with pytest.raises(RemoteCommandExecutionError):
remote_command_executor.run_remote_command("lctl get_param mdc.*.max_rpcs_in_flight | grep 64$")
with pytest.raises(RemoteCommandExecutionError):
remote_command_executor.run_remote_command("lctl get_param mdc.*.max_mod_rpcs_in_flight | grep 50$")


def _test_osu_benchmarks_pt2pt(
mpi_version, remote_command_executor, scheduler_commands, test_datadir, instance, slots_per_instance, partition=None
):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,8 @@ SharedStorage:
- MountDir: /shared
Name: name1
StorageType: Ebs
- MountDir: /fsx
Name: name2
StorageType: FsxLustre
FsxLustreSettings:
StorageCapacity: 1200