From 375b03239aa54152ad2e52d2d32ad848e86b8f70 Mon Sep 17 00:00:00 2001 From: Yuliia Miroshnychenko Date: Mon, 23 Oct 2023 12:42:34 +0200 Subject: [PATCH] [TEST]: 5449: Flow monitoring: Reroute: SYNC after recovering ISL SLA --- .../helpers/FlowHistoryConstants.groovy | 1 + .../spec/flows/FlowMonitoringSpec.groovy | 149 +++++++++++++++++- 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/FlowHistoryConstants.groovy b/src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/FlowHistoryConstants.groovy index e8fe9ad308f..544c6179fd9 100644 --- a/src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/FlowHistoryConstants.groovy +++ b/src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/FlowHistoryConstants.groovy @@ -6,6 +6,7 @@ class FlowHistoryConstants { public static String PARTIAL_UPDATE_ACTION = "Flow partial updating" public static String DELETE_ACTION = "Flow deleting" public static String REROUTE_ACTION = "Flow rerouting" + public static String SYNC_ACTION = "Flow paths sync" public static String UPDATE_SUCCESS = "Flow was updated successfully" public static String PARTIAL_UPDATE_ONLY_IN_DB = "Flow PATCH operation has been executed without the consecutive update." public static String CREATE_SUCCESS = "Flow was created successfully" diff --git a/src-java/testing/functional-tests/src/test/groovy/org/openkilda/functionaltests/spec/flows/FlowMonitoringSpec.groovy b/src-java/testing/functional-tests/src/test/groovy/org/openkilda/functionaltests/spec/flows/FlowMonitoringSpec.groovy index cb45789228c..67b5e3fc7bf 100644 --- a/src-java/testing/functional-tests/src/test/groovy/org/openkilda/functionaltests/spec/flows/FlowMonitoringSpec.groovy +++ b/src-java/testing/functional-tests/src/test/groovy/org/openkilda/functionaltests/spec/flows/FlowMonitoringSpec.groovy @@ -1,6 +1,16 @@ package org.openkilda.functionaltests.spec.flows +import static groovyx.gpars.GParsExecutorsPool.withPool +import static org.openkilda.functionaltests.helpers.FlowHelperV2.randomVlan +import static org.openkilda.functionaltests.helpers.FlowHistoryConstants.SYNC_ACTION + +import org.openkilda.functionaltests.helpers.model.Path import org.openkilda.functionaltests.model.stats.FlowStats +import org.openkilda.messaging.payload.flow.FlowState +import org.openkilda.northbound.dto.v2.flows.FlowRequestV2 +import org.openkilda.northbound.dto.v2.flows.FlowResponseV2 +import org.openkilda.testing.service.northbound.payloads.PathRequestParameter + import org.springframework.beans.factory.annotation.Autowired import static org.junit.jupiter.api.Assumptions.assumeTrue @@ -27,6 +37,7 @@ import org.openkilda.model.PathComputationStrategy import org.openkilda.testing.model.topology.TopologyDefinition.Isl import org.springframework.beans.factory.annotation.Value +import spock.lang.Ignore import spock.lang.Isolated import spock.lang.ResourceLock import spock.lang.See @@ -90,7 +101,6 @@ class FlowMonitoringSpec extends HealthCheckSpecification { .build()) and : "A flow with max_latency 210" - def createFlowTime = new Date() def flow = flowHelperV2.randomFlow(switchPair).tap { maxLatency = 210 pathComputationStrategy = PathComputationStrategy.MAX_LATENCY.toString() @@ -137,6 +147,135 @@ class FlowMonitoringSpec extends HealthCheckSpecification { initFeatureToggle && northbound.toggleFeature(initFeatureToggle) } + @ResourceLock(S42_TOGGLE) + @ResourceLock(FLOW_MON_TOGGLE) + @Ignore("https://github.com/telstra/open-kilda/issues/5449") + def "Able to detect several flows with MAX_LATENCY strategy when main path does not satisfy latency SLA\ + and successfully SYNC flows after ISL latency SLA recovering"() { + given: "flowLatencyMonitoringReactions is enabled in featureToggle" + and: "Disable s42 in featureToggle for generating flow-monitoring stats" + def initFeatureToggle = northbound.getFeatureToggles() + northbound.toggleFeature(FeatureTogglesDto.builder() + .flowLatencyMonitoringReactions(true) + .server42FlowRtt(false) + .build()) + + and : "Flows with max_latency 210 have been created successfully" + def flowRequest = flowHelperV2.randomFlow(switchPair).tap { + maxLatency = 210 + pathComputationStrategy = PathComputationStrategy.MAX_LATENCY.toString() + } + def flow1 = flowHelperV2.addFlow(flowRequest) + def flow2 = addFlowWithOverriddenVlans(flow1.flowId + "_flow2", flowRequest) + def flow3 = addFlowWithOverriddenVlans(flow1.flowId + "_flow3", flowRequest) + + //wait for generating some flow-monitoring stats + withPool { + [flow1.flowId, flow2.flowId, flow3.flowId].eachParallel { id -> + wait(flowSlaCheckIntervalSeconds + WAIT_OFFSET) { + flowStats.rttOf(id).get(FLOW_RTT, FORWARD).hasNonZeroValues() + } + } + } + + and: "Flows main and alternative paths have been collected successfully" + def flow1Isls = new Path(northbound.getFlowPath(flow1.flowId).forwardPath, topology).getInvolvedIsls() + def flow2Isls = new Path(northbound.getFlowPath(flow2.flowId).forwardPath, topology).getInvolvedIsls() + def flow3Isls = new Path(northbound.getFlowPath(flow3.flowId).forwardPath, topology).getInvolvedIsls() + + def alternativePathsIsls = northbound.getPaths([(PathRequestParameter.SRC_SWITCH) : flow1.source.switchId, + (PathRequestParameter.DST_SWITCH) : flow1.destination.switchId, + (PathRequestParameter.PATH_COMPUTATION_STRATEGY): PathRequestParameter.MAX_LATENCY, + (PathRequestParameter.MAX_LATENCY) : 210]) + .collectMany { it -> return [new Path(it.nodes, topology).getInvolvedIsls()] } + + when: "Common ISL does not satisfy SLA(update isl latency via db) and there is no alternative path to reroute flows" + def commonIslToSetDelay = flow1Isls.intersect(flow2Isls).intersect(flow2Isls.intersect(flow3Isls)).first() + assert commonIslToSetDelay, "Unfortunately flows don't share the same ISL. Cannot proceed with the further checking." + + List islToSetDelayAndDisableRerouting = alternativePathsIsls.findAll { !it.contains(commonIslToSetDelay)} + .collectMany { return [it.first()] }.flatten() + islToSetDelayAndDisableRerouting.each { + antiflap.portDown(it.srcSwitch.dpId, it.srcPort) + } + + def newLatency = (flowRequest.maxLatency + (flowRequest.maxLatency * flowLatencySlaThresholdPercent)).toInteger() + boolean isDelaySetUpForCommonIsl + String srcInterfaceName = commonIslToSetDelay.srcSwitch.name + "-" + commonIslToSetDelay.srcPort + isDelaySetUpForCommonIsl = true + String dstInterfaceName = commonIslToSetDelay.dstSwitch.name + "-" + commonIslToSetDelay.dstPort + lockKeeper.setLinkDelay(srcInterfaceName, newLatency) + lockKeeper.setLinkDelay(dstInterfaceName, newLatency) + withPool { + [flow1.flowId, flow2.flowId, flow3.flowId].eachParallel { id -> + wait(flowSlaCheckIntervalSeconds + WAIT_OFFSET) { + verifyLatencyInTsdb(id, newLatency) + } + } + } + then: "System detects that flowPathLatency doesn't satisfy max_latency and tries to reroute flows without success" + /** Steps before rerouting: check latency for a flow every 'kilda_flow_sla_check_interval_seconds'; + * wait 'kilda_flow_latency_sla_timeout_seconds'; + * recheck the flowLatency 'kilda_flow_sla_check_interval_seconds'; + * and then reroute the flow. + */ + withPool { + [flow1.flowId, flow2.flowId, flow3.flowId].eachParallel { id -> + wait(flowSlaCheckIntervalSeconds * 2 + flowLatencySlaTimeoutSeconds + WAIT_OFFSET) { + def history = northbound.getFlowHistory(id).last() + // Flow sync or flow reroute with reason "Flow latency become unhealthy" + assert history.getAction() == REROUTE_ACTION && history.details.contains("Flow latency become unhealthy") && + (history.payload.last().action in [REROUTE_FAIL] + && (history.payload.last().details.contains("Reroute is unsuccessful.") + || history.payload.last().details.contains("No path found. Can’t find a path from Node"))) + } + } + } + + withPool { + [flow1.flowId, flow2.flowId, flow3.flowId].eachParallel { id -> + if (northbound.getFlowHistory(id).last().payload.last().details.contains("Reroute is unsuccessful.")) { + wait(WAIT_OFFSET * 2) { assert northboundV2.getFlowStatus(id).status == FlowState.UP } + } else { + wait(WAIT_OFFSET * 2) { assert northboundV2.getFlowStatus(id).status == FlowState.DOWN } + } + } + } + + when: "The delay has been removed from shared ISL" + lockKeeper.cleanupLinkDelay(srcInterfaceName) + lockKeeper.cleanupLinkDelay(dstInterfaceName) + isDelaySetUpForCommonIsl = false + + then: "SYNC operation has been proceeded successfully for all flows" + //https://github.com/telstra/open-kilda/issues/5451 - unsuccessful SYNC (no event) + withPool { + [flow1.flowId, flow2.flowId, flow3.flowId].eachParallel { id -> + wait(flowSlaCheckIntervalSeconds * 2 + flowLatencySlaTimeoutSeconds + WAIT_OFFSET) { + def history = northbound.getFlowHistory(id).last() + assert history.getAction() == SYNC_ACTION && history.details.contains("Performing flow paths sync operation on NB request") + } + } + } + and: "All flows are in 'Up' status" + //now only for one flow SYNC operation has been executed successfully, the rest of flow is stuck in 'In_progress' status + withPool { + [flow1.flowId, flow2.flowId, flow3.flowId].eachParallel { id -> + wait(WAIT_OFFSET * 2) { assert northboundV2.getFlowStatus(id).status == FlowState.UP } + } + } + + cleanup: + isDelaySetUpForCommonIsl && lockKeeper.cleanupLinkDelay(srcInterfaceName) + isDelaySetUpForCommonIsl && lockKeeper.cleanupLinkDelay(dstInterfaceName) + islToSetDelayAndDisableRerouting.each { + antiflap.portUp(it.srcSwitch.dpId, it.srcPort) + } + //as a SYNC operation wasn't successful, we need to execute an additional manual setup for "In_Progress" flows to delete them + [flow1, flow2, flow3].findAll().each { flow -> flowHelperV2.deleteFlow(flow.flowId) } + initFeatureToggle && northbound.toggleFeature(initFeatureToggle) + } + @ResourceLock(S42_TOGGLE) @ResourceLock(FLOW_MON_TOGGLE) def "System doesn't try to reroute a MAX_LATENCY flow when a flow path doesn't satisfy latency SLA \ @@ -211,6 +350,14 @@ and flowLatencyMonitoringReactions is disabled in featureToggle"() { assert Math.abs(expectedNs - actual) <= expectedNs * 0.3 //less than 0.3 is unstable on jenkins } + FlowResponseV2 addFlowWithOverriddenVlans(String flowId, FlowRequestV2 flowRequest) { + flowHelperV2.addFlow(flowRequest.tap { + it.flowId = flowId + it.source.vlanId = randomVlan() + it.destination.vlanId = randomVlan() + }) + } + def cleanupSpec() { islsToBreak.each { getAntiflap().portUp(it.srcSwitch.dpId, it.srcPort) } wait(getDiscoveryInterval() + WAIT_OFFSET) {