diff --git a/tests/observability/metrics/conftest.py b/tests/observability/metrics/conftest.py index d3fc8a8ca0..5141e5a2ed 100644 --- a/tests/observability/metrics/conftest.py +++ b/tests/observability/metrics/conftest.py @@ -44,7 +44,6 @@ validate_vmi_sync_total_reported_and_positive, vnic_info_from_vm_or_vmi, ) -from tests.observability.utils import validate_metrics_value from tests.utils import create_vms, start_stress_on_vm from utilities import console from utilities.constants import Images @@ -86,7 +85,7 @@ get_pod_by_name_prefix, unique_name, ) -from utilities.monitoring import get_metrics_value +from utilities.monitoring import get_metrics_value, validate_metrics_value from utilities.network import assert_ping_successful, get_ip_from_vm_or_virt_handler_pod, ping from utilities.ssp import verify_ssp_pod_is_running from utilities.storage import ( diff --git a/tests/observability/metrics/test_aaq_metrics.py b/tests/observability/metrics/test_aaq_metrics.py index fc66874cc8..9e56a58dfc 100644 --- a/tests/observability/metrics/test_aaq_metrics.py +++ b/tests/observability/metrics/test_aaq_metrics.py @@ -4,7 +4,7 @@ timestamp_to_seconds, validate_values_from_kube_application_aware_resourcequota_metric, ) -from tests.observability.utils import validate_metrics_value +from utilities.monitoring import validate_metrics_value pytestmark = [ pytest.mark.usefixtures( diff --git a/tests/observability/metrics/test_general_metrics.py b/tests/observability/metrics/test_general_metrics.py index b5b83cad1c..11f6ee2baa 100644 --- a/tests/observability/metrics/test_general_metrics.py +++ b/tests/observability/metrics/test_general_metrics.py @@ -7,7 +7,7 @@ from libs.net.cluster import is_ipv6_single_stack_cluster from tests.observability.metrics.constants import KUBEVIRT_VMI_NODE_CPU_AFFINITY from tests.observability.metrics.utils import validate_vmi_node_cpu_affinity_with_prometheus -from tests.observability.utils import validate_metrics_value +from utilities.monitoring import validate_metrics_value from utilities.virt import VirtualMachineForTests, fedora_vm_body, running_vm KUBEVIRT_VM_TAG = f"{Resource.ApiGroup.KUBEVIRT_IO}/vm" diff --git a/tests/observability/metrics/test_metrics.py b/tests/observability/metrics/test_metrics.py index ca85435108..df051907e6 100644 --- a/tests/observability/metrics/test_metrics.py +++ b/tests/observability/metrics/test_metrics.py @@ -9,8 +9,8 @@ assert_vm_metric_virt_handler_pod, compare_kubevirt_vmi_info_metric_with_vm_info, ) -from tests.observability.utils import validate_metrics_value from utilities.constants.monitoring import KUBEVIRT_HCO_HYPERCONVERGED_CR_EXISTS +from utilities.monitoring import validate_metrics_value pytestmark = [pytest.mark.post_upgrade, pytest.mark.sno] diff --git a/tests/observability/metrics/test_migration_metrics.py b/tests/observability/metrics/test_migration_metrics.py index 93f38ad63b..aa6666379f 100644 --- a/tests/observability/metrics/test_migration_metrics.py +++ b/tests/observability/metrics/test_migration_metrics.py @@ -13,7 +13,7 @@ timestamp_to_seconds, wait_for_non_empty_metrics_value, ) -from tests.observability.utils import validate_metrics_value +from utilities.monitoring import validate_metrics_value LOGGER = logging.getLogger(__name__) diff --git a/tests/observability/metrics/test_ssp_metrics.py b/tests/observability/metrics/test_ssp_metrics.py index c68afd8c9a..c9cbeefb13 100644 --- a/tests/observability/metrics/test_ssp_metrics.py +++ b/tests/observability/metrics/test_ssp_metrics.py @@ -7,12 +7,12 @@ validate_metric_value_with_round_down, validate_metric_value_within_range, ) -from tests.observability.utils import validate_metrics_value from utilities.constants.components import ( SSP_OPERATOR, VIRT_TEMPLATE_VALIDATOR, ) from utilities.hco import ResourceEditorValidateHCOReconcile +from utilities.monitoring import validate_metrics_value from utilities.virt import VirtualMachineForTests KUBEVIRT_SSP_OPERATOR_UP = "kubevirt_ssp_operator_up" diff --git a/tests/observability/metrics/test_vms_metrics.py b/tests/observability/metrics/test_vms_metrics.py index d4e3852f62..c71e5be0af 100644 --- a/tests/observability/metrics/test_vms_metrics.py +++ b/tests/observability/metrics/test_vms_metrics.py @@ -31,7 +31,6 @@ validate_vmi_sync_total_reported_and_positive, validate_vnic_info, ) -from tests.observability.utils import validate_metrics_value from utilities.constants.pytest import QUARANTINED from utilities.constants.storage import ( CAPACITY, @@ -44,7 +43,7 @@ ) from utilities.constants.virt import MIGRATION_POLICY_VM_LABEL from utilities.infra import get_node_selector_dict -from utilities.monitoring import get_metrics_value +from utilities.monitoring import get_metrics_value, validate_metrics_value from utilities.virt import VirtualMachineForTests, fedora_vm_body, running_vm LOGGER = logging.getLogger(__name__) diff --git a/tests/observability/upgrade/test_upgrade_observability.py b/tests/observability/upgrade/test_upgrade_observability.py index 7f744159c9..b344b756c9 100644 --- a/tests/observability/upgrade/test_upgrade_observability.py +++ b/tests/observability/upgrade/test_upgrade_observability.py @@ -1,9 +1,9 @@ import pytest from tests.observability.constants import KUBEVIRT_VMI_NUMBER_OF_OUTDATED -from tests.observability.utils import validate_metrics_value from tests.upgrade_params import IUO_UPGRADE_TEST_DEPENDENCY_NODE_ID from utilities.constants.pytest import DEPENDENCY_SCOPE_SESSION +from utilities.monitoring import validate_metrics_value @pytest.mark.cnv_upgrade diff --git a/tests/observability/utils.py b/tests/observability/utils.py index a1edbbab07..96a60e04fd 100644 --- a/tests/observability/utils.py +++ b/tests/observability/utils.py @@ -1,46 +1,13 @@ -import datetime import logging from ocp_utilities.monitoring import Prometheus -from timeout_sampler import TimeoutExpiredError, TimeoutSampler from tests.observability.constants import SSP_COMMON_TEMPLATES_MODIFICATION_REVERTED -from utilities.constants.timeouts import ( - TIMEOUT_4MIN, - TIMEOUT_15SEC, -) -from utilities.monitoring import get_metrics_value LOGGER = logging.getLogger(__name__) ALLOW_ALERTS_ON_HEALTHY_CLUSTER_LIST = [SSP_COMMON_TEMPLATES_MODIFICATION_REVERTED] -def validate_metrics_value( - prometheus: Prometheus, metric_name: str, expected_value: str, timeout: int = TIMEOUT_4MIN -) -> None: - samples = TimeoutSampler( - wait_timeout=timeout, - sleep=TIMEOUT_15SEC, - func=get_metrics_value, - prometheus=prometheus, - metrics_name=metric_name, - ) - sample = None - comparison_values_log = {} - try: - for sample in samples: - if sample: - comparison_values_log[datetime.datetime.now()] = ( - f"metric: {metric_name} value is: {sample}, the expected value is {expected_value}" - ) - if sample == expected_value: - LOGGER.info("Metrics value matches the expected value!") - return - except TimeoutExpiredError: - LOGGER.error(f"Metrics value: {sample}, expected: {expected_value}, comparison log: {comparison_values_log}") - raise - - def verify_no_listed_alerts_on_cluster(prometheus: Prometheus, alerts_list: list[str]) -> None: """ It gets a list of alerts and verifies that none of them are firing on a cluster. diff --git a/utilities/monitoring.py b/utilities/monitoring.py index 1ba5df7bfe..4faac382f8 100644 --- a/utilities/monitoring.py +++ b/utilities/monitoring.py @@ -1,7 +1,12 @@ +import datetime import logging +from typing import TYPE_CHECKING from timeout_sampler import TimeoutExpiredError, TimeoutSampler +if TYPE_CHECKING: + from ocp_utilities.monitoring import Prometheus + from utilities.constants.monitoring import ( FIRING_STATE, KUBEVIRT_HYPERCONVERGED_OPERATOR_HEALTH_STATUS, @@ -177,6 +182,43 @@ def get_metrics_value(prometheus, metrics_name): return 0 +def validate_metrics_value( + prometheus: Prometheus, metric_name: str, expected_value: str | int, timeout: int = TIMEOUT_5MIN +) -> None: + """Wait until the metric matches the expected value. + + Args: + prometheus: Prometheus instance. + metric_name: Name of the metric to query. + expected_value: Expected value to match. Use str for emitted values (e.g. "0"), + int 0 for absent/not-emitted metrics (get_metrics_value returns int 0 when absent). + timeout: Maximum wait time in seconds. + + Raises: + TimeoutExpiredError: If the metric does not match within timeout. + """ + samples = TimeoutSampler( + wait_timeout=timeout, + sleep=TIMEOUT_5SEC, + func=get_metrics_value, + prometheus=prometheus, + metrics_name=metric_name, + ) + sample = None + comparison_values_log = {} + try: + for sample in samples: + comparison_values_log[datetime.datetime.now()] = ( + f"metric: {metric_name} value is: {sample}, the expected value is {expected_value}" + ) + if sample == expected_value: + LOGGER.info(f"Metrics value matches the expected value: {expected_value}") + return + except TimeoutExpiredError: + LOGGER.error(f"Metrics value: {sample}, expected: {expected_value}, comparison log: {comparison_values_log}") + raise + + def wait_for_gauge_metrics_value(prometheus, query, expected_value, timeout=TIMEOUT_5MIN): samples = TimeoutSampler( wait_timeout=timeout, diff --git a/utilities/unittests/test_monitoring.py b/utilities/unittests/test_monitoring.py index 98d0812570..088f16bde3 100644 --- a/utilities/unittests/test_monitoring.py +++ b/utilities/unittests/test_monitoring.py @@ -13,6 +13,7 @@ get_metrics_value, validate_alert_cnv_labels, validate_alerts, + validate_metrics_value, wait_for_alert, wait_for_firing_alert_clean_up, wait_for_gauge_metrics_value, @@ -351,6 +352,52 @@ def test_get_metrics_value_no_data(self): assert result == 0 +class TestValidateMetricsValue: + """Test cases for validate_metrics_value function""" + + @patch("utilities.monitoring.TimeoutSampler") + def test_matches_emitted_string_value(self, mock_sampler_cls): + """Test matching when metric is emitted with a string value.""" + mock_sampler_cls.return_value = iter(["0"]) + validate_metrics_value(prometheus=MagicMock(), metric_name="test_metric", expected_value="0") + + @patch("utilities.monitoring.TimeoutSampler") + def test_matches_absent_metric_with_int_zero(self, mock_sampler_cls): + """Test matching absent metric (int 0) with expected_value=0.""" + mock_sampler_cls.return_value = iter([0]) + validate_metrics_value(prometheus=MagicMock(), metric_name="test_metric", expected_value=0) + + @patch("utilities.monitoring.TimeoutSampler") + def test_absent_metric_does_not_match_string_zero(self, mock_sampler_cls): + """Test that absent metric (int 0) does NOT match expected_value='0'.""" + + def raise_after_samples(*args, **kwargs): + def _iter(): + yield 0 + raise TimeoutExpiredError("Timeout") + + return _iter() + + mock_sampler_cls.side_effect = raise_after_samples + with pytest.raises(TimeoutExpiredError): + validate_metrics_value(prometheus=MagicMock(), metric_name="test_metric", expected_value="0") + + @patch("utilities.monitoring.TimeoutSampler") + def test_timeout_when_value_does_not_match(self, mock_sampler_cls): + """Test timeout when metric value never matches expected.""" + + def raise_after_samples(*args, **kwargs): + def _iter(): + yield "5" + raise TimeoutExpiredError("Timeout") + + return _iter() + + mock_sampler_cls.side_effect = raise_after_samples + with pytest.raises(TimeoutExpiredError): + validate_metrics_value(prometheus=MagicMock(), metric_name="test_metric", expected_value="0") + + class TestWaitForGaugeMetricsValue: """Test cases for wait_for_gauge_metrics_value function"""